Merge pull request #58 from karthikuj/release-sasori

First release changes

README.md

@@ -123,9 +123,9 @@ The `crawler` section contains settings related to the behavior of the crawler.
     - **enabled**: (boolean) Specifies whether basic authentication is enabled.
     - **username**: (string) Username for basic authentication. Required if `enabled` is `true`.
     - **password**: (string) Password for basic authentication. Required if `enabled` is `true`.
-  - **scriptAuth**: (object) Configuration for script-based authentication.
-    - **enabled**: (boolean) Specifies whether script-based authentication is enabled.
-    - **pptrRecording**: (string) Path to Puppeteer recording script for authentication. Required if `enabled` is `true`.
+  - **recorderAuth**: (object) Configuration for recorder based authentication.
+    - **enabled**: (boolean) Specifies whether recorder based authentication is enabled.
+    - **pptrRecording**: (string) Path to Puppeteer recording for authentication. Required if `enabled` is `true`.
 
 Example:
 
@@ -146,7 +146,7 @@ Example:
         "username": "user",
         "password": "password"
       },
-      "scriptAuth": {
+      "recorderAuth": {
         "enabled": false,
         "pptrRecording": "/path/to/pptr/recording.json"
       }

config/config.json

@@ -27,7 +27,7 @@
                 "username": "username",
                 "password": "password"
             },
-            "scriptAuth": {
+            "recorderAuth": {
                 "enabled": false,
                 "pptrRecording": "/path/to/pptrRecording.json"
             }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "sasori-crawl",
-  "version": "0.0.1",
+  "version": "0.2.0",
   "description": "Sasori is a dynamic web crawler powered by Puppeteer, designed for lightning-fast endpoint discovery.",
   "main": "bin/index.js",
   "author": "Karthik UJ",

src/browser/browser.js

@@ -26,6 +26,7 @@ class Browser {
       args.push(
           `--proxy-server=${configJson.proxy.host}:${configJson.proxy.port}`,
       );
+      args.push('--ignore-certificate-errors');
     }
     if (configJson.slowMo) {
       browserConfig.slowMo = configJson.slowMo;

src/crawler/crawler.js

@@ -271,7 +271,7 @@ class Crawler {
    */
   async startAuthentication(browser, page) {
     this.authInProgress = true;
-    await authenticate(browser, page, path.resolve(__dirname, this.crawlerConfig.authentication.scriptAuth.pptrRecording));
+    await authenticate(browser, page, path.resolve(__dirname, this.crawlerConfig.authentication.recorderAuth.pptrRecording));
     this.authInProgress = false;
     await this.maximizeViewport(page);
   }
@@ -423,7 +423,7 @@ class Crawler {
     console.log(chalk.greenBright(`[INFO] Scope manager started successfully!`));
 
     // Start authentication if enabled.
-    if (this.crawlerConfig.authentication.scriptAuth && this.crawlerConfig.authentication.scriptAuth.enabled) {
+    if (this.crawlerConfig.authentication.recorderAuth && this.crawlerConfig.authentication.recorderAuth.enabled) {
       console.log(chalk.greenBright(`[INFO] Running initial authentication...`));
       await this.startAuthentication(browser, page);
     }

src/models/configModel.js

@@ -35,9 +35,9 @@ const basicAuthModel = Joi.object({
 }).required();
 
 /**
- * Script auth model for pptr script based auth configurations.
+ * Recorder auth model for pptr recorder based auth configurations.
  */
-const scriptAuthModel = Joi.object({
+const recorderAuthModel = Joi.object({
   enabled: Joi.boolean().required(),
   pptrRecording: Joi.string().when('enabled', {
     is: true,
@@ -51,7 +51,7 @@ const scriptAuthModel = Joi.object({
  */
 const authenticationModel = Joi.object({
   basicAuth: basicAuthModel,
-  scriptAuth: scriptAuthModel,
+  recorderAuth: recorderAuthModel,
 });
 
 /**