diff --git a/examples/Firecrawl Example Flow.json b/examples/Firecrawl Example Flow.json new file mode 100644 index 0000000..9899aa7 --- /dev/null +++ b/examples/Firecrawl Example Flow.json @@ -0,0 +1 @@ +{"id":"c427e888-97ec-461e-89e0-949e3199c815","data":{"nodes":[{"id":"File-SX5Zi","type":"genericNode","position":{"x":243.04974018776431,"y":-419.3520777543521},"data":{"type":"File","node":{"template":{"_type":"Component","path":{"trace_as_metadata":true,"file_path":"c427e888-97ec-461e-89e0-949e3199c815/pageOptions.json","fileTypes":["txt","md","mdx","csv","json","yaml","yml","xml","html","htm","pdf","docx","py","sh","sql","js","ts","tsx"],"list":false,"required":false,"placeholder":"","show":true,"value":"","name":"path","display_name":"Path","advanced":false,"dynamic":false,"info":"Supported file types: txt, md, mdx, csv, json, yaml, yml, xml, html, htm, pdf, docx, py, sh, sql, js, ts, tsx","title_case":false,"type":"file"},"code":{"type":"code","required":true,"placeholder":"","list":false,"show":true,"multiline":true,"value":"from pathlib import Path\n\nfrom langflow.base.data.utils import TEXT_FILE_TYPES, parse_text_file_to_data\nfrom langflow.custom import Component\nfrom langflow.io import BoolInput, FileInput, Output\nfrom langflow.schema import Data\n\n\nclass FileComponent(Component):\n display_name = \"File\"\n description = \"A generic file loader.\"\n icon = \"file-text\"\n\n inputs = [\n FileInput(\n name=\"path\",\n display_name=\"Path\",\n file_types=TEXT_FILE_TYPES,\n info=f\"Supported file types: {', '.join(TEXT_FILE_TYPES)}\",\n ),\n BoolInput(\n name=\"silent_errors\",\n display_name=\"Silent Errors\",\n advanced=True,\n info=\"If true, errors will not raise an exception.\",\n ),\n ]\n\n outputs = [\n Output(display_name=\"Data\", name=\"data\", method=\"load_file\"),\n ]\n\n def load_file(self) -> Data:\n if not self.path:\n raise ValueError(\"Please, upload a file to use this component.\")\n resolved_path = self.resolve_path(self.path)\n silent_errors = self.silent_errors\n\n extension = Path(resolved_path).suffix[1:].lower()\n\n if extension == \"doc\":\n raise ValueError(\"doc files are not supported. Please save as .docx\")\n if extension not in TEXT_FILE_TYPES:\n raise ValueError(f\"Unsupported file type: {extension}\")\n\n data = parse_text_file_to_data(resolved_path, silent_errors)\n self.status = data if data else \"No data\"\n return data or Data()\n","fileTypes":[],"file_path":"","password":false,"name":"code","advanced":true,"dynamic":true,"info":"","load_from_db":false,"title_case":false},"silent_errors":{"trace_as_metadata":true,"list":false,"required":false,"placeholder":"","show":true,"value":false,"name":"silent_errors","display_name":"Silent Errors","advanced":true,"dynamic":false,"info":"If true, errors will not raise an exception.","title_case":false,"type":"bool"}},"description":"A generic file loader.","icon":"file-text","base_classes":["Data"],"display_name":"File","documentation":"","custom_fields":{},"output_types":[],"pinned":false,"conditional_paths":[],"frozen":false,"outputs":[{"types":["Data"],"selected":"Data","name":"data","display_name":"Data","method":"load_file","value":"__UNDEFINED__","cache":true,"hidden":false}],"field_order":["path","silent_errors"],"beta":false,"edited":false},"id":"File-SX5Zi"},"selected":false,"width":384,"height":301,"positionAbsolute":{"x":243.04974018776431,"y":-419.3520777543521},"dragging":false},{"id":"ParseData-McOEp","type":"genericNode","position":{"x":1084.5020622168036,"y":-494.5559384566343},"data":{"type":"ParseData","node":{"template":{"_type":"Component","data":{"trace_as_input":true,"trace_as_metadata":true,"list":false,"required":false,"placeholder":"","show":true,"value":"","name":"data","display_name":"Data","advanced":false,"input_types":["Data"],"dynamic":false,"info":"The data to convert to text.","title_case":false,"type":"other"},"code":{"type":"code","required":true,"placeholder":"","list":false,"show":true,"multiline":true,"value":"from langflow.custom import Component\nfrom langflow.helpers.data import data_to_text\nfrom langflow.io import DataInput, MultilineInput, Output, StrInput\nfrom langflow.schema.message import Message\n\n\nclass ParseDataComponent(Component):\n display_name = \"Parse Data\"\n description = \"Convert Data into plain text following a specified template.\"\n icon = \"braces\"\n\n inputs = [\n DataInput(name=\"data\", display_name=\"Data\", info=\"The data to convert to text.\"),\n MultilineInput(\n name=\"template\",\n display_name=\"Template\",\n info=\"The template to use for formatting the data. It can contain the keys {text}, {data} or any other key in the Data.\",\n value=\"{text}\",\n ),\n StrInput(name=\"sep\", display_name=\"Separator\", advanced=True, value=\"\\n\"),\n ]\n\n outputs = [\n Output(display_name=\"Text\", name=\"text\", method=\"parse_data\"),\n ]\n\n def parse_data(self) -> Message:\n data = self.data if isinstance(self.data, list) else [self.data]\n template = self.template\n\n result_string = data_to_text(template, data, sep=self.sep)\n self.status = result_string\n return Message(text=result_string)\n","fileTypes":[],"file_path":"","password":false,"name":"code","advanced":true,"dynamic":true,"info":"","load_from_db":false,"title_case":false},"sep":{"trace_as_metadata":true,"load_from_db":false,"list":false,"required":false,"placeholder":"","show":true,"value":"\n","name":"sep","display_name":"Separator","advanced":true,"dynamic":false,"info":"","title_case":false,"type":"str"},"template":{"trace_as_input":true,"multiline":true,"trace_as_metadata":true,"load_from_db":false,"list":false,"required":false,"placeholder":"","show":true,"value":"{data}","name":"template","display_name":"Template","advanced":false,"input_types":["Message"],"dynamic":false,"info":"The template to use for formatting the data. It can contain the keys {text}, {data} or any other key in the Data.","title_case":false,"type":"str"}},"description":"Convert Data into plain text following a specified template.","icon":"braces","base_classes":["Message"],"display_name":"Parse Data","documentation":"","custom_fields":{},"output_types":[],"pinned":false,"conditional_paths":[],"frozen":false,"outputs":[{"types":["Message"],"selected":"Message","name":"text","display_name":"Text","method":"parse_data","value":"__UNDEFINED__","cache":true,"hidden":false}],"field_order":["data","template","sep"],"beta":false,"edited":false},"id":"ParseData-McOEp"},"selected":false,"width":384,"height":385,"positionAbsolute":{"x":1084.5020622168036,"y":-494.5559384566343},"dragging":false},{"id":"TextOutput-Ngxnj","type":"genericNode","position":{"x":1505.800647501932,"y":-415.3394124672299},"data":{"type":"TextOutput","node":{"template":{"_type":"Component","code":{"type":"code","required":true,"placeholder":"","list":false,"show":true,"multiline":true,"value":"from langflow.base.io.text import TextComponent\nfrom langflow.io import MessageTextInput, Output\nfrom langflow.schema.message import Message\n\n\nclass TextOutputComponent(TextComponent):\n display_name = \"Text Output\"\n description = \"Display a text output in the Playground.\"\n icon = \"type\"\n\n inputs = [\n MessageTextInput(\n name=\"input_value\",\n display_name=\"Text\",\n info=\"Text to be passed as output.\",\n ),\n ]\n outputs = [\n Output(display_name=\"Text\", name=\"text\", method=\"text_response\"),\n ]\n\n def text_response(self) -> Message:\n message = Message(\n text=self.input_value,\n )\n self.status = self.input_value\n return message\n","fileTypes":[],"file_path":"","password":false,"name":"code","advanced":true,"dynamic":true,"info":"","load_from_db":false,"title_case":false},"input_value":{"trace_as_input":true,"trace_as_metadata":true,"load_from_db":false,"list":false,"required":false,"placeholder":"","show":true,"value":"","name":"input_value","display_name":"Text","advanced":false,"input_types":["Message"],"dynamic":false,"info":"Text to be passed as output.","title_case":false,"type":"str"}},"description":"Display a text output in the Playground.","icon":"type","base_classes":["Message"],"display_name":"Text Output","documentation":"","custom_fields":{},"output_types":[],"pinned":false,"conditional_paths":[],"frozen":false,"outputs":[{"types":["Message"],"selected":"Message","name":"text","display_name":"Text","method":"text_response","value":"__UNDEFINED__","cache":true}],"field_order":["input_value"],"beta":false,"edited":false},"id":"TextOutput-Ngxnj"},"selected":false,"width":384,"height":309,"positionAbsolute":{"x":1505.800647501932,"y":-415.3394124672299},"dragging":false},{"id":"File-JLESg","type":"genericNode","position":{"x":238.4366870991455,"y":289.24705740316784},"data":{"type":"File","node":{"template":{"_type":"Component","path":{"trace_as_metadata":true,"file_path":"c427e888-97ec-461e-89e0-949e3199c815/search-pageOptions.json","fileTypes":["txt","md","mdx","csv","json","yaml","yml","xml","html","htm","pdf","docx","py","sh","sql","js","ts","tsx"],"list":false,"required":false,"placeholder":"","show":true,"value":"","name":"path","display_name":"Path","advanced":false,"dynamic":false,"info":"Supported file types: txt, md, mdx, csv, json, yaml, yml, xml, html, htm, pdf, docx, py, sh, sql, js, ts, tsx","title_case":false,"type":"file"},"code":{"type":"code","required":true,"placeholder":"","list":false,"show":true,"multiline":true,"value":"from pathlib import Path\n\nfrom langflow.base.data.utils import TEXT_FILE_TYPES, parse_text_file_to_data\nfrom langflow.custom import Component\nfrom langflow.io import BoolInput, FileInput, Output\nfrom langflow.schema import Data\n\n\nclass FileComponent(Component):\n display_name = \"File\"\n description = \"A generic file loader.\"\n icon = \"file-text\"\n\n inputs = [\n FileInput(\n name=\"path\",\n display_name=\"Path\",\n file_types=TEXT_FILE_TYPES,\n info=f\"Supported file types: {', '.join(TEXT_FILE_TYPES)}\",\n ),\n BoolInput(\n name=\"silent_errors\",\n display_name=\"Silent Errors\",\n advanced=True,\n info=\"If true, errors will not raise an exception.\",\n ),\n ]\n\n outputs = [\n Output(display_name=\"Data\", name=\"data\", method=\"load_file\"),\n ]\n\n def load_file(self) -> Data:\n if not self.path:\n raise ValueError(\"Please, upload a file to use this component.\")\n resolved_path = self.resolve_path(self.path)\n silent_errors = self.silent_errors\n\n extension = Path(resolved_path).suffix[1:].lower()\n\n if extension == \"doc\":\n raise ValueError(\"doc files are not supported. Please save as .docx\")\n if extension not in TEXT_FILE_TYPES:\n raise ValueError(f\"Unsupported file type: {extension}\")\n\n data = parse_text_file_to_data(resolved_path, silent_errors)\n self.status = data if data else \"No data\"\n return data or Data()\n","fileTypes":[],"file_path":"","password":false,"name":"code","advanced":true,"dynamic":true,"info":"","load_from_db":false,"title_case":false},"silent_errors":{"trace_as_metadata":true,"list":false,"required":false,"placeholder":"","show":true,"value":false,"name":"silent_errors","display_name":"Silent Errors","advanced":true,"dynamic":false,"info":"If true, errors will not raise an exception.","title_case":false,"type":"bool"}},"description":"A generic file loader.","icon":"file-text","base_classes":["Data"],"display_name":"File","documentation":"","custom_fields":{},"output_types":[],"pinned":false,"conditional_paths":[],"frozen":false,"outputs":[{"types":["Data"],"selected":"Data","name":"data","display_name":"Data","method":"load_file","value":"__UNDEFINED__","cache":true,"hidden":false}],"field_order":["path","silent_errors"],"beta":false,"edited":false},"id":"File-JLESg"},"selected":false,"width":384,"height":301,"positionAbsolute":{"x":238.4366870991455,"y":289.24705740316784},"dragging":false},{"id":"ParseData-TEdlQ","type":"genericNode","position":{"x":1090.754096139909,"y":201.60504494634858},"data":{"type":"ParseData","node":{"template":{"_type":"Component","data":{"trace_as_input":true,"trace_as_metadata":true,"list":false,"required":false,"placeholder":"","show":true,"value":"","name":"data","display_name":"Data","advanced":false,"input_types":["Data"],"dynamic":false,"info":"The data to convert to text.","title_case":false,"type":"other"},"code":{"type":"code","required":true,"placeholder":"","list":false,"show":true,"multiline":true,"value":"from langflow.custom import Component\nfrom langflow.helpers.data import data_to_text\nfrom langflow.io import DataInput, MultilineInput, Output, StrInput\nfrom langflow.schema.message import Message\n\n\nclass ParseDataComponent(Component):\n display_name = \"Parse Data\"\n description = \"Convert Data into plain text following a specified template.\"\n icon = \"braces\"\n\n inputs = [\n DataInput(name=\"data\", display_name=\"Data\", info=\"The data to convert to text.\"),\n MultilineInput(\n name=\"template\",\n display_name=\"Template\",\n info=\"The template to use for formatting the data. It can contain the keys {text}, {data} or any other key in the Data.\",\n value=\"{text}\",\n ),\n StrInput(name=\"sep\", display_name=\"Separator\", advanced=True, value=\"\\n\"),\n ]\n\n outputs = [\n Output(display_name=\"Text\", name=\"text\", method=\"parse_data\"),\n ]\n\n def parse_data(self) -> Message:\n data = self.data if isinstance(self.data, list) else [self.data]\n template = self.template\n\n result_string = data_to_text(template, data, sep=self.sep)\n self.status = result_string\n return Message(text=result_string)\n","fileTypes":[],"file_path":"","password":false,"name":"code","advanced":true,"dynamic":true,"info":"","load_from_db":false,"title_case":false},"sep":{"trace_as_metadata":true,"load_from_db":false,"list":false,"required":false,"placeholder":"","show":true,"value":"\n","name":"sep","display_name":"Separator","advanced":true,"dynamic":false,"info":"","title_case":false,"type":"str"},"template":{"trace_as_input":true,"multiline":true,"trace_as_metadata":true,"load_from_db":false,"list":false,"required":false,"placeholder":"","show":true,"value":"{data}","name":"template","display_name":"Template","advanced":false,"input_types":["Message"],"dynamic":false,"info":"The template to use for formatting the data. It can contain the keys {text}, {data} or any other key in the Data.","title_case":false,"type":"str"}},"description":"Convert Data into plain text following a specified template.","icon":"braces","base_classes":["Message"],"display_name":"Parse Data","documentation":"","custom_fields":{},"output_types":[],"pinned":false,"conditional_paths":[],"frozen":false,"outputs":[{"types":["Message"],"selected":"Message","name":"text","display_name":"Text","method":"parse_data","value":"__UNDEFINED__","cache":true,"hidden":false}],"field_order":["data","template","sep"],"beta":false,"edited":false},"id":"ParseData-TEdlQ"},"selected":false,"width":384,"height":385,"dragging":false,"positionAbsolute":{"x":1090.754096139909,"y":201.60504494634858}},{"id":"TextOutput-R1OfL","type":"genericNode","position":{"x":1509.5816961086375,"y":272.15901325169693},"data":{"type":"TextOutput","node":{"template":{"_type":"Component","code":{"type":"code","required":true,"placeholder":"","list":false,"show":true,"multiline":true,"value":"from langflow.base.io.text import TextComponent\nfrom langflow.io import MessageTextInput, Output\nfrom langflow.schema.message import Message\n\n\nclass TextOutputComponent(TextComponent):\n display_name = \"Text Output\"\n description = \"Display a text output in the Playground.\"\n icon = \"type\"\n\n inputs = [\n MessageTextInput(\n name=\"input_value\",\n display_name=\"Text\",\n info=\"Text to be passed as output.\",\n ),\n ]\n outputs = [\n Output(display_name=\"Text\", name=\"text\", method=\"text_response\"),\n ]\n\n def text_response(self) -> Message:\n message = Message(\n text=self.input_value,\n )\n self.status = self.input_value\n return message\n","fileTypes":[],"file_path":"","password":false,"name":"code","advanced":true,"dynamic":true,"info":"","load_from_db":false,"title_case":false},"input_value":{"trace_as_input":true,"trace_as_metadata":true,"load_from_db":false,"list":false,"required":false,"placeholder":"","show":true,"value":"","name":"input_value","display_name":"Text","advanced":false,"input_types":["Message"],"dynamic":false,"info":"Text to be passed as output.","title_case":false,"type":"str"}},"description":"Display a text output in the Playground.","icon":"type","base_classes":["Message"],"display_name":"Text Output","documentation":"","custom_fields":{},"output_types":[],"pinned":false,"conditional_paths":[],"frozen":false,"outputs":[{"types":["Message"],"selected":"Message","name":"text","display_name":"Text","method":"text_response","value":"__UNDEFINED__","cache":true}],"field_order":["input_value"],"beta":false,"edited":false},"id":"TextOutput-R1OfL"},"selected":false,"width":384,"height":309,"positionAbsolute":{"x":1509.5816961086375,"y":272.15901325169693},"dragging":false},{"id":"FirecrawlScrapeApi-T7J1L","type":"genericNode","position":{"x":662.4544848030141,"y":-696.8447460781325},"data":{"type":"FirecrawlScrapeApi","node":{"template":{"_type":"CustomComponent","extractorOptions":{"type":"Data","required":false,"placeholder":"","list":false,"show":true,"multiline":false,"fileTypes":[],"file_path":"","password":false,"name":"extractorOptions","display_name":"Extractor Options","advanced":false,"dynamic":false,"info":"The extractor options to send with the request.","load_from_db":false,"title_case":false},"pageOptions":{"type":"Data","required":false,"placeholder":"","list":false,"show":true,"multiline":false,"fileTypes":[],"file_path":"","password":false,"name":"pageOptions","display_name":"Page Options","advanced":false,"dynamic":false,"info":"The page options to send with the request.","load_from_db":false,"title_case":false},"api_key":{"type":"str","required":true,"placeholder":"","list":false,"show":true,"multiline":false,"fileTypes":[],"file_path":"","password":true,"name":"api_key","display_name":"API Key","advanced":false,"dynamic":false,"info":"The API key to use Firecrawl API.","load_from_db":false,"title_case":false,"input_types":["Text"],"value":""},"code":{"type":"code","required":true,"placeholder":"","list":false,"show":true,"multiline":true,"value":"from typing import Optional\nfrom firecrawl.firecrawl import FirecrawlApp\nfrom langflow.custom import CustomComponent\nfrom langflow.schema import Data\nfrom langflow.services.database.models.base import orjson_dumps\nimport json\n\nclass FirecrawlScrapeApi(CustomComponent):\n display_name: str = \"FirecrawlScrapeApi\"\n description: str = \"Firecrawl Scrape API.\"\n output_types: list[str] = [\"Document\"]\n documentation: str = \"https://docs.firecrawl.dev/api-reference/endpoint/scrape\"\n field_config = {\n \"api_key\": {\n \"display_name\": \"API Key\",\n \"field_type\": \"str\",\n \"required\": True,\n \"password\": True,\n \"info\": \"The API key to use Firecrawl API.\",\n },\n \"url\": {\n \"display_name\": \"URL\",\n \"field_type\": \"str\",\n \"required\": True,\n \"info\": \"The URL to scrape.\",\n },\n \"timeout\": {\n \"display_name\": \"Timeout\",\n \"info\": \"Timeout in milliseconds for the request.\",\n \"field_type\": \"int\",\n \"default_value\": 10000,\n },\n \"pageOptions\": {\n \"display_name\": \"Page Options\",\n \"info\": \"The page options to send with the request.\",\n },\n \"extractorOptions\": {\n \"display_name\": \"Extractor Options\",\n \"info\": \"The extractor options to send with the request.\",\n },\n }\n\n def build(\n self,\n api_key: str,\n url: str,\n timeout: Optional[int] = 10000,\n pageOptions: Optional[Data] = None,\n extractorOptions: Optional[Data] = None,\n ) -> Data:\n if extractorOptions:\n extractor_options_dict = extractorOptions.__dict__['data']['text']\n else:\n extractor_options_dict = {}\n\n if pageOptions:\n page_options_dict = pageOptions.__dict__['data']['text']\n else:\n page_options_dict = {}\n\n app = FirecrawlApp(api_key=api_key)\n results = app.scrape_url(\n url,\n {\n \"timeout\": str(timeout),\n \"extractorOptions\": extractor_options_dict,\n \"pageOptions\": page_options_dict,\n },\n )\n\n record = Data(data=results)\n return record\n","fileTypes":[],"file_path":"","password":false,"name":"code","advanced":true,"dynamic":true,"info":"","load_from_db":false,"title_case":false},"timeout":{"type":"int","required":false,"placeholder":"","list":false,"show":true,"multiline":false,"value":10000,"fileTypes":[],"file_path":"","password":false,"name":"timeout","display_name":"Timeout","advanced":false,"dynamic":false,"info":"Timeout in milliseconds for the request.","load_from_db":false,"title_case":false},"url":{"type":"str","required":true,"placeholder":"","list":false,"show":true,"multiline":false,"fileTypes":[],"file_path":"","password":false,"name":"url","display_name":"URL","advanced":false,"dynamic":false,"info":"The URL to scrape.","load_from_db":false,"title_case":false,"input_types":["Text"],"value":"firecrawl.dev"}},"description":"Firecrawl Scrape API.","base_classes":["Data"],"display_name":"FirecrawlScrapeApi","documentation":"https://docs.firecrawl.dev/api-reference/endpoint/scrape","custom_fields":{"api_key":null,"url":null,"timeout":null,"pageOptions":null,"extractorOptions":null},"output_types":["Data"],"pinned":false,"conditional_paths":[],"frozen":false,"outputs":[{"types":["Data"],"selected":"Data","name":"data","hidden":false,"display_name":"Data","method":null,"value":"__UNDEFINED__","cache":true}],"field_order":["api_key","url","timeout","pageOptions","extractorOptions"],"beta":false,"edited":true},"id":"FirecrawlScrapeApi-T7J1L","description":"Firecrawl Scrape API.","display_name":"FirecrawlScrapeApi"},"selected":false,"width":384,"height":581,"positionAbsolute":{"x":662.4544848030141,"y":-696.8447460781325},"dragging":false},{"id":"FirecrawlCrawlApi-zklFP","type":"genericNode","position":{"x":663.8060779623452,"y":-87.86502220070001},"data":{"type":"FirecrawlCrawlApi","node":{"template":{"_type":"CustomComponent","crawlerOptions":{"type":"Data","required":false,"placeholder":"","list":false,"show":true,"multiline":false,"fileTypes":[],"file_path":"","password":false,"name":"crawlerOptions","display_name":"Crawler Options","advanced":false,"dynamic":false,"info":"Options for the crawler behavior.","load_from_db":false,"title_case":false},"pageOptions":{"type":"Data","required":false,"placeholder":"","list":false,"show":true,"multiline":false,"fileTypes":[],"file_path":"","password":false,"name":"pageOptions","display_name":"Page Options","advanced":false,"dynamic":false,"info":"The page options to send with the request.","load_from_db":false,"title_case":false},"api_key":{"type":"str","required":true,"placeholder":"","list":false,"show":true,"multiline":false,"fileTypes":[],"file_path":"","password":true,"name":"api_key","display_name":"API Key","advanced":false,"dynamic":false,"info":"The API key to use Firecrawl API.","load_from_db":false,"title_case":false,"input_types":["Text"],"value":""},"code":{"type":"code","required":true,"placeholder":"","list":false,"show":true,"multiline":true,"value":"from typing import Optional\nfrom firecrawl.firecrawl import FirecrawlApp\nfrom langflow.custom import CustomComponent\nfrom langflow.schema import Data\nimport json\nimport uuid\n\nclass FirecrawlCrawlApi(CustomComponent):\n display_name: str = \"FirecrawlCrawlApi\"\n description: str = \"Firecrawl Crawl API.\"\n output_types: list[str] = [\"Document\"]\n documentation: str = \"https://docs.firecrawl.dev/api-reference/endpoint/crawl\"\n field_config = {\n \"api_key\": {\n \"display_name\": \"API Key\",\n \"field_type\": \"str\",\n \"required\": True,\n \"password\": True,\n \"info\": \"The API key to use Firecrawl API.\",\n },\n \"url\": {\n \"display_name\": \"URL\",\n \"field_type\": \"str\",\n \"required\": True,\n \"info\": \"The base URL to start crawling from.\",\n },\n \"timeout\": {\n \"display_name\": \"Timeout\",\n \"field_type\": \"int\",\n \"info\": \"The timeout in milliseconds.\",\n },\n \"crawlerOptions\": {\n \"display_name\": \"Crawler Options\",\n \"info\": \"Options for the crawler behavior.\",\n },\n \"pageOptions\": {\n \"display_name\": \"Page Options\",\n \"info\": \"The page options to send with the request.\",\n },\n \"idempotency_key\": {\n \"display_name\": \"Idempotency Key\",\n \"field_type\": \"str\",\n \"info\": \"Optional idempotency key to ensure unique requests.\",\n },\n }\n\n def build(\n self,\n api_key: str,\n url: str,\n timeout: Optional[int] = 30000,\n crawlerOptions: Optional[Data] = None,\n pageOptions: Optional[Data] = None,\n idempotency_key: Optional[str] = None,\n ) -> Data:\n if crawlerOptions:\n crawler_options_dict = crawlerOptions.__dict__['data']['text']\n else:\n crawler_options_dict = {}\n\n if pageOptions:\n page_options_dict = pageOptions.__dict__['data']['text']\n else:\n page_options_dict = {}\n\n if not idempotency_key:\n idempotency_key = str(uuid.uuid4())\n\n app = FirecrawlApp(api_key=api_key)\n crawl_result = app.crawl_url(\n url,\n {\n \"crawlerOptions\": crawler_options_dict,\n \"pageOptions\": page_options_dict,\n },\n True,\n int(timeout / 1000),\n idempotency_key\n )\n\n records = Data(data={\"results\": crawl_result})\n return records\n","fileTypes":[],"file_path":"","password":false,"name":"code","advanced":true,"dynamic":true,"info":"","load_from_db":false,"title_case":false},"idempotency_key":{"type":"str","required":false,"placeholder":"","list":false,"show":true,"multiline":false,"fileTypes":[],"file_path":"","password":false,"name":"idempotency_key","display_name":"Idempotency Key","advanced":false,"dynamic":false,"info":"Optional idempotency key to ensure unique requests.","load_from_db":false,"title_case":false,"input_types":["Text"]},"timeout":{"type":"int","required":false,"placeholder":"","list":false,"show":true,"multiline":false,"value":30000,"fileTypes":[],"file_path":"","password":false,"name":"timeout","display_name":"Timeout","advanced":false,"dynamic":false,"info":"The timeout in milliseconds.","load_from_db":false,"title_case":false},"url":{"type":"str","required":true,"placeholder":"","list":false,"show":true,"multiline":false,"fileTypes":[],"file_path":"","password":false,"name":"url","display_name":"URL","advanced":false,"dynamic":false,"info":"The base URL to start crawling from.","load_from_db":false,"title_case":false,"input_types":["Text"],"value":"firecrawl.dev"}},"description":"Firecrawl Crawl API.","base_classes":["Data"],"display_name":"FirecrawlCrawlApi","documentation":"https://docs.firecrawl.dev/api-reference/endpoint/crawl","custom_fields":{"api_key":null,"url":null,"timeout":null,"crawlerOptions":null,"pageOptions":null,"idempotency_key":null},"output_types":["Data"],"pinned":false,"conditional_paths":[],"frozen":false,"outputs":[{"types":["Data"],"selected":"Data","name":"data","hidden":false,"display_name":"Data","method":null,"value":"__UNDEFINED__","cache":true}],"field_order":["api_key","url","timeout","crawlerOptions","pageOptions","idempotency_key"],"beta":false,"edited":false},"id":"FirecrawlCrawlApi-zklFP"},"selected":false,"width":384,"height":675,"positionAbsolute":{"x":663.8060779623452,"y":-87.86502220070001},"dragging":false}],"edges":[{"source":"ParseData-McOEp","sourceHandle":"{œdataTypeœ:œParseDataœ,œidœ:œParseData-McOEpœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}","target":"TextOutput-Ngxnj","targetHandle":"{œfieldNameœ:œinput_valueœ,œidœ:œTextOutput-Ngxnjœ,œinputTypesœ:[œMessageœ],œtypeœ:œstrœ}","data":{"targetHandle":{"fieldName":"input_value","id":"TextOutput-Ngxnj","inputTypes":["Message"],"type":"str"},"sourceHandle":{"dataType":"ParseData","id":"ParseData-McOEp","name":"text","output_types":["Message"]}},"id":"reactflow__edge-ParseData-McOEp{œdataTypeœ:œParseDataœ,œidœ:œParseData-McOEpœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}-TextOutput-Ngxnj{œfieldNameœ:œinput_valueœ,œidœ:œTextOutput-Ngxnjœ,œinputTypesœ:[œMessageœ],œtypeœ:œstrœ}","className":""},{"source":"ParseData-TEdlQ","sourceHandle":"{œdataTypeœ:œParseDataœ,œidœ:œParseData-TEdlQœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}","target":"TextOutput-R1OfL","targetHandle":"{œfieldNameœ:œinput_valueœ,œidœ:œTextOutput-R1OfLœ,œinputTypesœ:[œMessageœ],œtypeœ:œstrœ}","data":{"targetHandle":{"fieldName":"input_value","id":"TextOutput-R1OfL","inputTypes":["Message"],"type":"str"},"sourceHandle":{"dataType":"ParseData","id":"ParseData-TEdlQ","name":"text","output_types":["Message"]}},"id":"reactflow__edge-ParseData-TEdlQ{œdataTypeœ:œParseDataœ,œidœ:œParseData-TEdlQœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}-TextOutput-R1OfL{œfieldNameœ:œinput_valueœ,œidœ:œTextOutput-R1OfLœ,œinputTypesœ:[œMessageœ],œtypeœ:œstrœ}","className":""},{"source":"File-SX5Zi","sourceHandle":"{œdataTypeœ:œFileœ,œidœ:œFile-SX5Ziœ,œnameœ:œdataœ,œoutput_typesœ:[œDataœ]}","target":"FirecrawlScrapeApi-T7J1L","targetHandle":"{œfieldNameœ:œpageOptionsœ,œidœ:œFirecrawlScrapeApi-T7J1Lœ,œinputTypesœ:null,œtypeœ:œDataœ}","data":{"targetHandle":{"fieldName":"pageOptions","id":"FirecrawlScrapeApi-T7J1L","inputTypes":null,"type":"Data"},"sourceHandle":{"dataType":"File","id":"File-SX5Zi","name":"data","output_types":["Data"]}},"id":"reactflow__edge-File-SX5Zi{œdataTypeœ:œFileœ,œidœ:œFile-SX5Ziœ,œnameœ:œdataœ,œoutput_typesœ:[œDataœ]}-FirecrawlScrapeApi-T7J1L{œfieldNameœ:œpageOptionsœ,œidœ:œFirecrawlScrapeApi-T7J1Lœ,œinputTypesœ:null,œtypeœ:œDataœ}","className":""},{"source":"FirecrawlScrapeApi-T7J1L","sourceHandle":"{œdataTypeœ:œFirecrawlScrapeApiœ,œidœ:œFirecrawlScrapeApi-T7J1Lœ,œnameœ:œdataœ,œoutput_typesœ:[œDataœ]}","target":"ParseData-McOEp","targetHandle":"{œfieldNameœ:œdataœ,œidœ:œParseData-McOEpœ,œinputTypesœ:[œDataœ],œtypeœ:œotherœ}","data":{"targetHandle":{"fieldName":"data","id":"ParseData-McOEp","inputTypes":["Data"],"type":"other"},"sourceHandle":{"dataType":"FirecrawlScrapeApi","id":"FirecrawlScrapeApi-T7J1L","name":"data","output_types":["Data"]}},"id":"reactflow__edge-FirecrawlScrapeApi-T7J1L{œdataTypeœ:œFirecrawlScrapeApiœ,œidœ:œFirecrawlScrapeApi-T7J1Lœ,œnameœ:œdataœ,œoutput_typesœ:[œDataœ]}-ParseData-McOEp{œfieldNameœ:œdataœ,œidœ:œParseData-McOEpœ,œinputTypesœ:[œDataœ],œtypeœ:œotherœ}","className":""},{"source":"File-JLESg","sourceHandle":"{œdataTypeœ:œFileœ,œidœ:œFile-JLESgœ,œnameœ:œdataœ,œoutput_typesœ:[œDataœ]}","target":"FirecrawlCrawlApi-zklFP","targetHandle":"{œfieldNameœ:œpageOptionsœ,œidœ:œFirecrawlCrawlApi-zklFPœ,œinputTypesœ:null,œtypeœ:œDataœ}","data":{"targetHandle":{"fieldName":"pageOptions","id":"FirecrawlCrawlApi-zklFP","inputTypes":null,"type":"Data"},"sourceHandle":{"dataType":"File","id":"File-JLESg","name":"data","output_types":["Data"]}},"id":"reactflow__edge-File-JLESg{œdataTypeœ:œFileœ,œidœ:œFile-JLESgœ,œnameœ:œdataœ,œoutput_typesœ:[œDataœ]}-FirecrawlCrawlApi-zklFP{œfieldNameœ:œpageOptionsœ,œidœ:œFirecrawlCrawlApi-zklFPœ,œinputTypesœ:null,œtypeœ:œDataœ}","className":""},{"source":"FirecrawlCrawlApi-zklFP","sourceHandle":"{œdataTypeœ:œFirecrawlCrawlApiœ,œidœ:œFirecrawlCrawlApi-zklFPœ,œnameœ:œdataœ,œoutput_typesœ:[œDataœ]}","target":"ParseData-TEdlQ","targetHandle":"{œfieldNameœ:œdataœ,œidœ:œParseData-TEdlQœ,œinputTypesœ:[œDataœ],œtypeœ:œotherœ}","data":{"targetHandle":{"fieldName":"data","id":"ParseData-TEdlQ","inputTypes":["Data"],"type":"other"},"sourceHandle":{"dataType":"FirecrawlCrawlApi","id":"FirecrawlCrawlApi-zklFP","name":"data","output_types":["Data"]}},"id":"reactflow__edge-FirecrawlCrawlApi-zklFP{œdataTypeœ:œFirecrawlCrawlApiœ,œidœ:œFirecrawlCrawlApi-zklFPœ,œnameœ:œdataœ,œoutput_typesœ:[œDataœ]}-ParseData-TEdlQ{œfieldNameœ:œdataœ,œidœ:œParseData-TEdlQœ,œinputTypesœ:[œDataœ],œtypeœ:œotherœ}","className":""}],"viewport":{"x":56.55720248973125,"y":386.38105360596376,"zoom":0.4115593875913944}},"description":"Firecrawl example flow with Crawl and Scrape components.","name":"Firecrawl API Example","last_tested_version":"1.0.3","endpoint_name":null,"is_component":false}