From 8bb3ebe763c1f5b254172e0da9242589cff76653 Mon Sep 17 00:00:00 2001 From: Coleman Kane Date: Mon, 27 Jun 2022 15:21:52 -0400 Subject: [PATCH 01/24] Update with newer cred file specification This suppresses a warning --- aws/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/main.tf b/aws/main.tf index b15c520..4b75534 100644 --- a/aws/main.tf +++ b/aws/main.tf @@ -1,6 +1,6 @@ provider "aws" { region = var.region - shared_credentials_file = "~/.aws/credentials" + shared_credentials_files = ["$HOME/.aws/credentials"] profile = "default" } From 67c50e7708a44acea0d91098124957766186f77b Mon Sep 17 00:00:00 2001 From: Coleman Kane Date: Mon, 27 Jun 2022 15:23:49 -0400 Subject: [PATCH 02/24] Update to use TypeDB rather than Grakn Grakn was deprecated in favor of TypeDB, and the old links are no longer active --- opencti_scripts/installer.sh | 48 ++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/opencti_scripts/installer.sh b/opencti_scripts/installer.sh index a2bbcbb..b62cd41 100644 --- a/opencti_scripts/installer.sh +++ b/opencti_scripts/installer.sh @@ -126,7 +126,7 @@ function check_apt_pkg { } # Function: check_service -# Checks if a service is active or nah. Matches Grakn service output. +# Checks if a service is active or nah. Matches TypeDB service output. # Parameters: # - $1: service to check function check_service { @@ -202,11 +202,11 @@ else quit_on_error echo "You are using an unsupported version of Ubuntu. Exiting." fi -# Grakn -grakn_bin_version="2.0.0-alpha-6" -grakn_console_version="2.0.0-alpha-4" -grakn_core_all_version="2.0.0-alpha-4" -grakn_core_server_version="2.0.0-alpha-4" +# TypeDB +typedb_bin_version="2.11.0" +typedb_console_version="2.11.0" +typedb_core_all_version="2.11.0" +typedb_core_server_version="2.11.0" # Minio minio_dir="/opt/minio/data" @@ -263,7 +263,7 @@ disable_service 'elasticsearch' disable_service 'redis-server' disable_service 'rabbitmq-server' disable_service 'minio' -disable_service 'grakn' +disable_service 'typedb' # The VMs we're running are not that big and we're going to quickly fill the system log with our work (and especially the connectors). This will max out the logs at 100M. echo "SystemMaxUse=100M" >> /etc/systemd/journald.conf @@ -304,34 +304,34 @@ check_apt_pkg "python3-pip" ${run_python} -m pip install --upgrade pip ${run_python} -m pip -q install --ignore-installed PyYAML -## Grakn -log_section_heading "Grakn" +## TypeDB +log_section_heading "TypeDB" sudo apt-key adv --keyserver keyserver.ubuntu.com --recv 8F3DA4B5E9AEF44C -sudo add-apt-repository 'deb [ arch=all ] https://repo.grakn.ai/repository/apt/ trusty main' +sudo add-apt-repository 'deb [ arch=all ] https://repo.vaticle.com/repository/apt/ trusty main' update_apt_pkg # apt-get install -y grakn-console=2.0.0-alpha-3 # Required dependency # apt-get install -y grakn-core-all -check_apt_pkg 'grakn-bin' "=${grakn_bin_version}" -check_apt_pkg 'grakn-core-server' "=${grakn_core_server_version}" -check_apt_pkg 'grakn-console' "=${grakn_console_version}" -check_apt_pkg 'grakn-core-all' "=${grakn_core_all_version}" +# check_apt_pkg 'grakn-bin' "=${grakn_bin_version}" +# check_apt_pkg 'grakn-core-server' "=${grakn_core_server_version}" +# check_apt_pkg 'grakn-console' "=${grakn_console_version}" +check_apt_pkg 'typedb-all' "=${typedb_core_all_version}" -### Create systemd unit file for Grakn -cat < /etc/systemd/system/grakn.service +### Create systemd unit file for TypeDB +cat < /etc/systemd/system/typedb.service [Unit] -Description=Grakn.AI Server daemon +Description=TypeDB Server daemon After=network.target [Service] -Type=forking -ExecStart=/usr/local/bin/grakn server start -ExecStop=/usr/local/bin/grakn server stop -ExecReload=/usr/local/bin/grakn server stop && /usr/local/bin/grakn server start -RemainAfterExit=yes +Type=simple +ExecStart=/usr/local/bin/typedb server +#ExecStop=/usr/local/bin/typedb server stop +#ExecReload=/usr/local/bin/typedb server stop && /usr/local/bin/typedb server start +#RemainAfterExit=yes [Install] WantedBy=multi-user.target EOT systemctl daemon-reload -enable_service 'grakn' +enable_service 'typedb' ## Elasticsearch log_section_heading "Elasticsearch" @@ -514,7 +514,7 @@ echo -e "${RMQ_user_list}" # Check status of services log_section_heading "Checking service statuses" check_service 'elasticsearch' -check_service 'grakn' +check_service 'typedb' check_service 'minio' check_service 'rabbitmq-server' check_service 'redis-server' From bd6fd8e3fd2ef94b04d3f17232998e327a5765d3 Mon Sep 17 00:00:00 2001 From: Coleman Kane Date: Mon, 27 Jun 2022 15:24:56 -0400 Subject: [PATCH 03/24] Update rabbitmq version and repositories Older version of rabbitmq no longer available, and old repository links are defunct --- opencti_scripts/installer.sh | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/opencti_scripts/installer.sh b/opencti_scripts/installer.sh index b62cd41..1ea1235 100644 --- a/opencti_scripts/installer.sh +++ b/opencti_scripts/installer.sh @@ -215,7 +215,7 @@ minio_dir="/opt/minio/data" redis_ver="6.0.5" # RabbitMQ -rabbitmq_ver="3.8.5-1" +rabbitmq_ver="3.8.34-1" rabbitmq_release_url="https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc" # OpenCTI @@ -471,13 +471,23 @@ enable_service 'redis-server' ## RabbitMQ log_section_heading "RabbitMQ" curl -fsSL "${rabbitmq_release_url}" | apt-key add - -tee /etc/apt/sources.list.d/bintray.rabbitmq.list < Date: Mon, 27 Jun 2022 15:26:04 -0400 Subject: [PATCH 04/24] Update to latest OpenCTI+connectors (5.3.7) The connectors were moved into various subdirectories, so additional logic was added to that part of the install & service code to ensure that the appropriate sub-paths are used to properly install and start them up at boot. --- opencti_scripts/connectors.sh | 58 ++++++++++++++++++----------------- opencti_scripts/installer.sh | 8 ++--- 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/opencti_scripts/connectors.sh b/opencti_scripts/connectors.sh index 130004c..8368f91 100644 --- a/opencti_scripts/connectors.sh +++ b/opencti_scripts/connectors.sh @@ -140,26 +140,26 @@ warn_user # This will only set up your instance for the connectors enabled. You must supply an API token (e.g., alienvault token) and enable the service. # It should be safe to run this after changing configs or enabling services. declare -A CONNECTORS; -CONNECTORS['alienvault']=0 -CONNECTORS['amitt']=0 -CONNECTORS['crowdstrike']=0 -CONNECTORS['cryptolaemus']=0 -CONNECTORS['cve']=1 -CONNECTORS['cyber-threat-coalition']=0 -CONNECTORS['cybercrime-tracker']=0 -CONNECTORS['export-file-csv']=1 -CONNECTORS['export-file-stix']=1 -CONNECTORS['hygiene']=0 -CONNECTORS['import-file-pdf-observables']=1 -CONNECTORS['import-file-stix']=1 -CONNECTORS['ipinfo']=0 -CONNECTORS['lastinfosec']=0 -CONNECTORS['malpedia']=0 -CONNECTORS['misp']=1 -CONNECTORS['mitre']=1 -CONNECTORS['opencti']=1 -CONNECTORS['valhalla']=0 -CONNECTORS['virustotal']=1 +CONNECTORS['external-import/alienvault']=0 +CONNECTORS['external-import/amitt']=0 +CONNECTORS['external-import/crowdstrike']=0 +CONNECTORS['external-import/cryptolaemus']=0 +CONNECTORS['external-import/cve']=1 +CONNECTORS['external-import/cyber-threat-coalition']=0 +CONNECTORS['external-import/cybercrime-tracker']=0 +CONNECTORS['internal-export-file/export-file-csv']=1 +CONNECTORS['internal-export-file/export-file-stix']=1 +CONNECTORS['internal-enrichment/hygiene']=0 +CONNECTORS['internal-import-file/import-document']=1 +CONNECTORS['internal-import-file/import-file-stix']=1 +CONNECTORS['internal-enrichment/ipinfo']=0 +CONNECTORS['external-import/lastinfosec']=0 +CONNECTORS['external-import/malpedia']=0 +CONNECTORS['external-import/misp']=1 +CONNECTORS['external-import/mitre']=1 +CONNECTORS['external-import/opencti']=1 +CONNECTORS['external-import/valhalla']=0 +CONNECTORS['internal-enrichment/virustotal']=1 echo "The following connectors will be installed:" for i in "${!CONNECTORS[@]}" @@ -200,16 +200,18 @@ do sed -i"" -e "s/id: 'ChangeMe'/id: '$(uuidgen -r | tr -d '\n' | tr '[:upper:]' '[:lower:]')'/g" "${opencti_connector_dir}/$i/src/config.yml" fi - if [[ ! -f "/etc/systemd/system/opencti-connector-$i.service" ]] + sbasename=$(basename "$i") + + if [[ ! -f "/etc/systemd/system/opencti-connector-$sbasename.service" ]] then - cat > /etc/systemd/system/opencti-connector-$i.service <<- EOT + cat > /etc/systemd/system/opencti-connector-$sbasename.service <<- EOT [Unit] Description=OpenCTI Connector - $i After=network.target [Service] Type=simple WorkingDirectory=${opencti_connector_dir}/$i/src -ExecStart=/usr/bin/python${python_ver} "${opencti_connector_dir}/$i/src/$i.py" +ExecStart=/usr/bin/python${python_ver} "${opencti_connector_dir}/$i/src/$sbasename.py" ExecReload=/bin/kill -s HUP \$MAINPID ExecStop=/bin/kill -s TERM \$MAINPID PrivateTmp=true @@ -219,16 +221,16 @@ WantedBy=multi-user.target EOT systemctl daemon-reload - systemctl start opencti-connector-$i.service + systemctl start opencti-connector-$sbasename.service fi - if [[ $(systemctl status --no-pager opencti-connector-$i.service | grep 'Active: active') ]] + if [[ $(systemctl status --no-pager opencti-connector-$sbasename.service | grep 'Active: active') ]] then - echo "opencti-connector-$i.service is already running, restarting due to config changes" - systemctl restart opencti-connector-$i.service + echo "opencti-connector-$sbasename.service is already running, restarting due to config changes" + systemctl restart opencti-connector-$sbasename.service fi - quit_on_error "Installing service for connector: $i" + quit_on_error "Installing service for connector: $sbasename" fi done diff --git a/opencti_scripts/installer.sh b/opencti_scripts/installer.sh index 1ea1235..07558a1 100644 --- a/opencti_scripts/installer.sh +++ b/opencti_scripts/installer.sh @@ -232,9 +232,9 @@ do esac done -opencti_ver="4.2.1" +opencti_ver="5.3.7" opencti_dir="/opt/opencti" -opencti_worker_count=2 +opencti_worker_count=4 # ########### # Main script @@ -541,8 +541,8 @@ echo "Changing owner of ${opencti_dir} to:" $(whoami)":"$(id -gn) chown -R $(whoami):$(id -gn) "${opencti_dir}" echo "OpenCTI: Installing Python dependencies" -${run_python} -m pip -q install -r "${opencti_dir}/connectors/export-file-stix/src/requirements.txt" -${run_python} -m pip -q install -r "${opencti_dir}/connectors/import-file-stix/src/requirements.txt" +${run_python} -m pip -q install -r "${opencti_dir}/connectors/internal-export-file/export-file-stix/src/requirements.txt" +${run_python} -m pip -q install -r "${opencti_dir}/connectors/internal-import-file/import-file-stix/src/requirements.txt" ${run_python} -m pip -q install -r "${opencti_dir}/src/python/requirements.txt" ${run_python} -m pip -q install -r "${opencti_dir}/worker/requirements.txt" ${run_python} -m pip install requests==2.25.0 From 003cf5c6f3663ebb86bb072dd607eee98ac562a3 Mon Sep 17 00:00:00 2001 From: Coleman Kane Date: Mon, 27 Jun 2022 19:40:32 -0400 Subject: [PATCH 05/24] Update redis to 7.0.2 --- opencti_scripts/installer.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opencti_scripts/installer.sh b/opencti_scripts/installer.sh index 07558a1..79e9c3e 100644 --- a/opencti_scripts/installer.sh +++ b/opencti_scripts/installer.sh @@ -212,7 +212,7 @@ typedb_core_server_version="2.11.0" minio_dir="/opt/minio/data" # Redis -redis_ver="6.0.5" +redis_ver="7.0.2" # RabbitMQ rabbitmq_ver="3.8.34-1" @@ -436,7 +436,7 @@ fi if [[ ! -f "/etc/redis/redis.conf" ]] then cp "redis-${redis_ver}/redis.conf" "/etc/redis/redis.conf" - sed -i 's/^supervised no/supervised systemd/' "/etc/redis/redis.conf" + sed -i 's/^\#\ supervised\ .*$/supervised auto/' "/etc/redis/redis.conf" chown redis:redis "/etc/redis/redis.conf" fi From c4a68a7ed5104f48b582d59df48a6b32054c9bc7 Mon Sep 17 00:00:00 2001 From: Coleman Kane Date: Mon, 27 Jun 2022 20:06:52 -0400 Subject: [PATCH 06/24] Update rabbitmq to 3.10.5-1 (latest) --- opencti_scripts/installer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opencti_scripts/installer.sh b/opencti_scripts/installer.sh index 79e9c3e..bd3683a 100644 --- a/opencti_scripts/installer.sh +++ b/opencti_scripts/installer.sh @@ -215,7 +215,7 @@ minio_dir="/opt/minio/data" redis_ver="7.0.2" # RabbitMQ -rabbitmq_ver="3.8.34-1" +rabbitmq_ver="3.10.5-1" rabbitmq_release_url="https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc" # OpenCTI From b832bdd31cd914970458629221e2237a61720f46 Mon Sep 17 00:00:00 2001 From: Coleman Kane Date: Tue, 28 Jun 2022 19:00:12 -0400 Subject: [PATCH 07/24] Update TypeDB to 2.11.1 --- opencti_scripts/installer.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/opencti_scripts/installer.sh b/opencti_scripts/installer.sh index bd3683a..921c47b 100644 --- a/opencti_scripts/installer.sh +++ b/opencti_scripts/installer.sh @@ -203,10 +203,10 @@ else fi # TypeDB -typedb_bin_version="2.11.0" -typedb_console_version="2.11.0" -typedb_core_all_version="2.11.0" -typedb_core_server_version="2.11.0" +typedb_bin_version="2.11.1" +typedb_console_version="2.11.1" +typedb_core_all_version="2.11.1" +typedb_core_server_version="2.11.1" # Minio minio_dir="/opt/minio/data" From 0c510bd5c9503a3c5f37bb7ec011b5be8cf19b7d Mon Sep 17 00:00:00 2001 From: Coleman Kane Date: Thu, 30 Jun 2022 08:28:19 -0400 Subject: [PATCH 08/24] Extend redis startup timeout to 30s to give it time to start --- opencti_scripts/installer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opencti_scripts/installer.sh b/opencti_scripts/installer.sh index 921c47b..98700e7 100644 --- a/opencti_scripts/installer.sh +++ b/opencti_scripts/installer.sh @@ -457,7 +457,7 @@ Type=notify ExecStart=/usr/local/bin/redis-server /etc/redis/redis.conf ExecStop=/usr/local/bin/redis-cli -p 6379 shutdown ExecReload=/bin/kill -USR2 \$MAINPID -TimeoutStartSec=10 +TimeoutStartSec=30 TimeoutStopSec=10 Restart=on-failure [Install] From 45c40f2ba98899f85d3f695070dbf2845ce8e09f Mon Sep 17 00:00:00 2001 From: Coleman Kane Date: Sat, 2 Jul 2022 12:20:03 -0400 Subject: [PATCH 09/24] Adjust service startup timeout limits When the database contains a lot of data, it can take longer to bring up both redis-server.service and opencti-server.service. The downstream dependencies (workers, connectors, etc...) need to better account for slow bring-up of those dependencies. --- opencti_scripts/connectors.sh | 5 +++++ opencti_scripts/installer.sh | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/opencti_scripts/connectors.sh b/opencti_scripts/connectors.sh index 8368f91..a94b978 100644 --- a/opencti_scripts/connectors.sh +++ b/opencti_scripts/connectors.sh @@ -208,7 +208,12 @@ do [Unit] Description=OpenCTI Connector - $i After=network.target +StartLimitBurst=30 +StartLimitInterval=0 + [Service] +RestartSec=20 +TimeoutStartSec=600 Type=simple WorkingDirectory=${opencti_connector_dir}/$i/src ExecStart=/usr/bin/python${python_ver} "${opencti_connector_dir}/$i/src/$sbasename.py" diff --git a/opencti_scripts/installer.sh b/opencti_scripts/installer.sh index 98700e7..ab511de 100644 --- a/opencti_scripts/installer.sh +++ b/opencti_scripts/installer.sh @@ -588,7 +588,12 @@ cat > /etc/systemd/system/opencti-worker@.service <<- EOT [Unit] Description=OpenCTI Worker daemon %i After=network.target opencti-server.service +StartLimitBurst=30 +StartLimitInterval=0 + [Service] +RestartSec=20 +TimeoutStartSec=600 Type=simple WorkingDirectory=${opencti_dir}/worker/ ExecStart=/usr/bin/${run_python} "${opencti_dir}/worker/worker.py" From 12219d2d307038fc1a5fd492195b6973b1bc7b99 Mon Sep 17 00:00:00 2001 From: Coleman Kane Date: Sat, 2 Jul 2022 12:21:41 -0400 Subject: [PATCH 10/24] Check for connector main script named "main.py", and use it if it exists Some of the connectors have started using "main.py" as their main execution script, rather than a Python script using the service's name as the filename. Check for that, and adapt the generated systemd unit definition as needed. --- opencti_scripts/connectors.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/opencti_scripts/connectors.sh b/opencti_scripts/connectors.sh index a94b978..66c3aab 100644 --- a/opencti_scripts/connectors.sh +++ b/opencti_scripts/connectors.sh @@ -201,6 +201,14 @@ do fi sbasename=$(basename "$i") + scriptname="$sbasename" + + # Some of the services use the service name as the main Python script name, while + # others have started using "main.py". This check attempts to discover which the + # service in question is using, and adapts to it, falling back on the old behavior + if [[ -f "${opencti_connector_dir}/$i/src/main.py" ]]; then + scriptname="main" + fi if [[ ! -f "/etc/systemd/system/opencti-connector-$sbasename.service" ]] then @@ -216,7 +224,7 @@ RestartSec=20 TimeoutStartSec=600 Type=simple WorkingDirectory=${opencti_connector_dir}/$i/src -ExecStart=/usr/bin/python${python_ver} "${opencti_connector_dir}/$i/src/$sbasename.py" +ExecStart=/usr/bin/python${python_ver} "${opencti_connector_dir}/$i/src/$scriptname.py" ExecReload=/bin/kill -s HUP \$MAINPID ExecStop=/bin/kill -s TERM \$MAINPID PrivateTmp=true From 52f6094cf91b398d9f3d8045dbe74dc535ac76ae Mon Sep 17 00:00:00 2001 From: Coleman Kane Date: Sat, 2 Jul 2022 19:31:44 -0400 Subject: [PATCH 11/24] Allow more time for redis to start before timeout --- opencti_scripts/installer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opencti_scripts/installer.sh b/opencti_scripts/installer.sh index ab511de..81b364a 100644 --- a/opencti_scripts/installer.sh +++ b/opencti_scripts/installer.sh @@ -457,7 +457,7 @@ Type=notify ExecStart=/usr/local/bin/redis-server /etc/redis/redis.conf ExecStop=/usr/local/bin/redis-cli -p 6379 shutdown ExecReload=/bin/kill -USR2 \$MAINPID -TimeoutStartSec=30 +TimeoutStartSec=900 TimeoutStopSec=10 Restart=on-failure [Install] From bc93c884a88c735264111bd5d256ab9f73f32e68 Mon Sep 17 00:00:00 2001 From: Coleman Kane Date: Wed, 6 Jul 2022 21:30:05 -0400 Subject: [PATCH 12/24] Add valhalla module When provided no API key (or empty string), it will download a bunch of free & open yara signatures hosted by Florian Roth. --- opencti_scripts/connectors.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opencti_scripts/connectors.sh b/opencti_scripts/connectors.sh index 66c3aab..34c2e03 100644 --- a/opencti_scripts/connectors.sh +++ b/opencti_scripts/connectors.sh @@ -158,7 +158,7 @@ CONNECTORS['external-import/malpedia']=0 CONNECTORS['external-import/misp']=1 CONNECTORS['external-import/mitre']=1 CONNECTORS['external-import/opencti']=1 -CONNECTORS['external-import/valhalla']=0 +CONNECTORS['external-import/valhalla']=1 CONNECTORS['internal-enrichment/virustotal']=1 echo "The following connectors will be installed:" From 081b06a23891e1f23dfb80240c3224ef41089563 Mon Sep 17 00:00:00 2001 From: Coleman Kane Date: Wed, 7 Sep 2022 10:29:40 -0400 Subject: [PATCH 13/24] Support Ubuntu 22.04 LTS --- opencti_scripts/connectors.sh | 4 ++++ opencti_scripts/installer.sh | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/opencti_scripts/connectors.sh b/opencti_scripts/connectors.sh index 34c2e03..f815fb8 100644 --- a/opencti_scripts/connectors.sh +++ b/opencti_scripts/connectors.sh @@ -120,6 +120,10 @@ if [[ ${ubuntu_version} == 18 ]] then python_ver="3.7" elif [[ ${ubuntu_version} == 20 ]] +then + # Using bionic since focal not avaialble yet for RabbitMQ + python_ver="3" +elif [[ ${ubuntu_version} == 22 ]] then # Using bionic since focal not avaialble yet for RabbitMQ python_ver="3" diff --git a/opencti_scripts/installer.sh b/opencti_scripts/installer.sh index 81b364a..d030606 100644 --- a/opencti_scripts/installer.sh +++ b/opencti_scripts/installer.sh @@ -194,6 +194,11 @@ then distro="bionic" run_python="python3.7" elif [[ ${ubuntu_version} == 20 ]] +then + # Using bionic since focal not avaialble yet for RabbitMQ + distro="bionic" + run_python="python3" +elif [[ ${ubuntu_version} == 22 ]] then # Using bionic since focal not avaialble yet for RabbitMQ distro="bionic" From b7df9196bfb5beda6cd02e20278eaaa14382bf3c Mon Sep 17 00:00:00 2001 From: Coleman Kane Date: Wed, 7 Sep 2022 10:30:43 -0400 Subject: [PATCH 14/24] Fix the installation of TypeDB, which breaks with typedb-all now --- opencti_scripts/installer.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/opencti_scripts/installer.sh b/opencti_scripts/installer.sh index d030606..39a89a0 100644 --- a/opencti_scripts/installer.sh +++ b/opencti_scripts/installer.sh @@ -208,7 +208,7 @@ else fi # TypeDB -typedb_bin_version="2.11.1" +typedb_bin_version="2.9.0" typedb_console_version="2.11.1" typedb_core_all_version="2.11.1" typedb_core_server_version="2.11.1" @@ -319,6 +319,8 @@ update_apt_pkg # check_apt_pkg 'grakn-bin' "=${grakn_bin_version}" # check_apt_pkg 'grakn-core-server' "=${grakn_core_server_version}" # check_apt_pkg 'grakn-console' "=${grakn_console_version}" +check_apt_pkg 'typedb-bin' "=${typedb_bin_version}" +check_apt_pkg 'typedb-server' "=${typedb_core_all_version}" check_apt_pkg 'typedb-all' "=${typedb_core_all_version}" ### Create systemd unit file for TypeDB From 6f1000d47123928e3d78a4660cdf3f7618de68ae Mon Sep 17 00:00:00 2001 From: Coleman Kane Date: Wed, 7 Sep 2022 10:31:12 -0400 Subject: [PATCH 15/24] Update to OpenCTI 5.3.10 --- opencti_scripts/installer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opencti_scripts/installer.sh b/opencti_scripts/installer.sh index 39a89a0..5f35cf4 100644 --- a/opencti_scripts/installer.sh +++ b/opencti_scripts/installer.sh @@ -237,7 +237,7 @@ do esac done -opencti_ver="5.3.7" +opencti_ver="5.3.10" opencti_dir="/opt/opencti" opencti_worker_count=4 From dc01b8a9a36d488717daa5650012ac16c0be91d2 Mon Sep 17 00:00:00 2001 From: Coleman Kane Date: Wed, 7 Sep 2022 10:31:41 -0400 Subject: [PATCH 16/24] Install latest elasticsearch 8.x This also requires some fixes to turn off the "security features", since it is for internal use only, these only add overhead --- opencti_scripts/installer.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/opencti_scripts/installer.sh b/opencti_scripts/installer.sh index 5f35cf4..95595a4 100644 --- a/opencti_scripts/installer.sh +++ b/opencti_scripts/installer.sh @@ -358,9 +358,12 @@ cat < /etc/logrotate.d/elasticsearch } EOT wget -qO - 'https://artifacts.elastic.co/GPG-KEY-elasticsearch' | apt-key add - -add-apt-repository "deb https://artifacts.elastic.co/packages/7.x/apt stable main" +add-apt-repository "deb https://artifacts.elastic.co/packages/8.x/apt stable main" update_apt_pkg check_apt_pkg 'elasticsearch' +sed -i 's|xpack.security.enabled: true|xpack.security.enabled: false|' /etc/elasticsearch/elasticsearch.yml +sed -i 's|xpack.security.enrollment.enabled: true|xpack.security.enrollment.enabled: false|' /etc/elasticsearch/elasticsearch.yml +sed -i 's|^ enabled: true| enabled: false|' /etc/elasticsearch/elasticsearch.yml enable_service 'elasticsearch' ## Minio From 2e742fa669dd8c0c8784faf2bbef05c3fd1cdab8 Mon Sep 17 00:00:00 2001 From: Coleman Kane Date: Wed, 7 Sep 2022 10:32:38 -0400 Subject: [PATCH 17/24] Check machine arch for correct minio download The minio server download should match the architecture we're installing on. This allows deployment on ARM (like Graviton) to work. --- opencti_scripts/installer.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/opencti_scripts/installer.sh b/opencti_scripts/installer.sh index 95595a4..479499b 100644 --- a/opencti_scripts/installer.sh +++ b/opencti_scripts/installer.sh @@ -368,7 +368,8 @@ enable_service 'elasticsearch' ## Minio log_section_heading "Minio" -wget --quiet -O minio https://dl.min.io/server/minio/release/linux-amd64/minio +my_minio_arch=`uname -m | sed s/aarch64/arm64/g | sed s/x86_64/amd64/g` +wget --quiet -O minio https://dl.min.io/server/minio/release/linux-${my_minio_arch}/minio chmod +x minio mv minio "/usr/local/bin/" if [[ ! -d "${minio_dir}" ]] From f3482096e842790388a3fd077ba6730654da8fb9 Mon Sep 17 00:00:00 2001 From: Coleman Kane Date: Wed, 7 Sep 2022 21:29:09 -0400 Subject: [PATCH 18/24] Pass the ${storage_bucket} variable down to the sub-scripts --- userdata/installation-wrapper-script.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/userdata/installation-wrapper-script.sh b/userdata/installation-wrapper-script.sh index 3dd2ed9..1a8b8b2 100644 --- a/userdata/installation-wrapper-script.sh +++ b/userdata/installation-wrapper-script.sh @@ -41,10 +41,10 @@ chmod +x /opt/${connectors_script_name} echo "Starting OpenCTI installation script" # Run the install script with the provided e-mail address. # AWS automatically runs the script as root, Azure doesn't. -sudo /opt/${install_script_name} -e "${login_email}" +sudo /usr/bin/env storage_bucket="${storage_bucket}" /opt/${install_script_name} -e "${login_email}" echo "OpenCTI installation script complete." echo "Starting OpenCTI connectors script." # Run the script without prompting the user (the default, `-p 0`, will prompt if the user wants to apply; this is less than ideal for an automated script). -sudo /opt/${connectors_script_name} -p 1 +sudo /usr/bin/env storage_bucket="${storage_bucket}" /opt/${connectors_script_name} -p 1 echo "OpenCTI wrapper script complete." From 17bea66f3cc2ef9763e7cb1a25bd2aa8729a95f9 Mon Sep 17 00:00:00 2001 From: Coleman Kane Date: Wed, 7 Sep 2022 21:33:06 -0400 Subject: [PATCH 19/24] Use S3 instead of Minio for blob storage This uses the same bucket that stores the installation scripts for artifact storage. --- opencti_scripts/installer.sh | 46 +----------------------------------- 1 file changed, 1 insertion(+), 45 deletions(-) diff --git a/opencti_scripts/installer.sh b/opencti_scripts/installer.sh index 479499b..9401c01 100644 --- a/opencti_scripts/installer.sh +++ b/opencti_scripts/installer.sh @@ -213,9 +213,6 @@ typedb_console_version="2.11.1" typedb_core_all_version="2.11.1" typedb_core_server_version="2.11.1" -# Minio -minio_dir="/opt/minio/data" - # Redis redis_ver="7.0.2" @@ -267,7 +264,6 @@ disable_service 'opencti-server' disable_service 'elasticsearch' disable_service 'redis-server' disable_service 'rabbitmq-server' -disable_service 'minio' disable_service 'typedb' # The VMs we're running are not that big and we're going to quickly fill the system log with our work (and especially the connectors). This will max out the logs at 100M. @@ -366,45 +362,6 @@ sed -i 's|xpack.security.enrollment.enabled: true|xpack.security.enrollment.enab sed -i 's|^ enabled: true| enabled: false|' /etc/elasticsearch/elasticsearch.yml enable_service 'elasticsearch' -## Minio -log_section_heading "Minio" -my_minio_arch=`uname -m | sed s/aarch64/arm64/g | sed s/x86_64/amd64/g` -wget --quiet -O minio https://dl.min.io/server/minio/release/linux-${my_minio_arch}/minio -chmod +x minio -mv minio "/usr/local/bin/" -if [[ ! -d "${minio_dir}" ]] -then - mkdir -p "${minio_dir}" -fi - -### From: https://github.com/minio/minio-service/blob/master/linux-systemd/minio.service -if [[ ! -f "/etc/default/minio" ]] -then - # .minio.access_key - RMINIOAK="$(openssl rand -hex 12)" - # .minio.secret_key - RMINIOSK="$(openssl rand -base64 25 | tr -d '/')" - cat > /etc/default/minio <<- EOT -# Volume to be used for MinIO server. -MINIO_VOLUMES="/opt/minio/data/" -# Use if you want to run MinIO on a custom port. -# MINIO_OPTS="--address :9199" -# Access Key of the server. -MINIO_ACCESS_KEY=${RMINIOAK} -# Secret key of the server. -MINIO_SECRET_KEY=${RMINIOSK} -EOT -else - RMINIOAK="$(grep -o 'MINIO_ACCESS_KEY=.*' /etc/default/minio | cut -f2- -d=)" - RMINIOSK="$(grep -o 'MINIO_SECRET_KEY=.*' /etc/default/minio | cut -f2- -d=)" -fi - -curl "https://raw.githubusercontent.com/minio/minio-service/master/linux-systemd/minio.service" -o "/etc/systemd/system/minio.service" -sed -i'' -e 's/User=minio-user/User=root/g' "/etc/systemd/system/minio.service" -sed -i'' -e 's/Group=minio-user/Group=root/g' "/etc/systemd/system/minio.service" -systemctl daemon-reload -enable_service 'minio' - ## Redis log_section_heading "Redis" update_apt_pkg @@ -536,7 +493,6 @@ echo -e "${RMQ_user_list}" log_section_heading "Checking service statuses" check_service 'elasticsearch' check_service 'typedb' -check_service 'minio' check_service 'rabbitmq-server' check_service 'redis-server' @@ -567,7 +523,7 @@ RADMINTOKEN="$(uuidgen -r | tr -d '\n' | tr '[:upper:]' '[:lower:]')" echo "OpenCTI: Copy proper configs" # Take default configuration and fill in our values. -cat ${opencti_dir}/config/default.json | jq ".app.admin.email=\"${opencti_email}\" | .app.admin.password=\"${RADMINPASS}\" | .app.admin.token=\"${RADMINTOKEN}\" | .minio.access_key=\"${RMINIOAK}\" | .minio.secret_key=\"${RMINIOSK}\" | .rabbitmq.username=\"${RRMQUNAME}\" | .rabbitmq.password=\"${RRMQPASS}\"" > ${opencti_dir}/config/production.json +cat ${opencti_dir}/config/default.json | jq ".app.admin.email=\"${opencti_email}\" | .app.admin.password=\"${RADMINPASS}\" | .app.admin.token=\"${RADMINTOKEN}\" | .minio.bucket_name=\"${storage_bucket}\" | .minio.endpoint=\"s3.amazonaws.com\" | .minio.port=443 | .minio.use_ssl=true | .minio.access_key=\"\" | .minio.secret_key=\"\" | .minio.use_aws_role=true | .rabbitmq.username=\"${RRMQUNAME}\" | .rabbitmq.password=\"${RRMQPASS}\"" > ${opencti_dir}/config/production.json echo "OpenCTI: Create unit file" cat > /etc/systemd/system/opencti-server.service <<- EOT From df8304c926ba68c757776a5cb71a3923be3e24ab Mon Sep 17 00:00:00 2001 From: Coleman Kane Date: Sun, 11 Sep 2022 19:00:00 -0400 Subject: [PATCH 20/24] Bump to 5.3.12 --- opencti_scripts/installer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opencti_scripts/installer.sh b/opencti_scripts/installer.sh index 9401c01..eb80131 100644 --- a/opencti_scripts/installer.sh +++ b/opencti_scripts/installer.sh @@ -234,7 +234,7 @@ do esac done -opencti_ver="5.3.10" +opencti_ver="5.3.12" opencti_dir="/opt/opencti" opencti_worker_count=4 From 11598cea005a4b37bba359364451391744e0f0f6 Mon Sep 17 00:00:00 2001 From: Coleman Kane Date: Tue, 4 Oct 2022 08:40:21 -0400 Subject: [PATCH 21/24] Update to 5.3.16 --- opencti_scripts/installer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opencti_scripts/installer.sh b/opencti_scripts/installer.sh index eb80131..8a2c6bd 100644 --- a/opencti_scripts/installer.sh +++ b/opencti_scripts/installer.sh @@ -234,7 +234,7 @@ do esac done -opencti_ver="5.3.12" +opencti_ver="5.3.16" opencti_dir="/opt/opencti" opencti_worker_count=4 From f53f30e205c712bf2bd45598c8fe20b727e5ee8e Mon Sep 17 00:00:00 2001 From: Coleman Kane Date: Sun, 9 Oct 2022 11:02:25 -0400 Subject: [PATCH 22/24] Update to 5.3.17 and update redis to latest as well --- opencti_scripts/installer.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opencti_scripts/installer.sh b/opencti_scripts/installer.sh index 8a2c6bd..9b96d37 100644 --- a/opencti_scripts/installer.sh +++ b/opencti_scripts/installer.sh @@ -214,7 +214,7 @@ typedb_core_all_version="2.11.1" typedb_core_server_version="2.11.1" # Redis -redis_ver="7.0.2" +redis_ver="7.0.5" # RabbitMQ rabbitmq_ver="3.10.5-1" @@ -234,7 +234,7 @@ do esac done -opencti_ver="5.3.16" +opencti_ver="5.3.17" opencti_dir="/opt/opencti" opencti_worker_count=4 From 3c6c2cd82a0a22def7e329ea0ab0b46654d4c5a6 Mon Sep 17 00:00:00 2001 From: Coleman Kane Date: Sun, 9 Oct 2022 12:12:25 -0400 Subject: [PATCH 23/24] Update for newer terraform s3 code Bucket acls and versioning is now in its own resource, and the aws_s3_bucket_object is now just aws_s3_object --- aws/storage.tf | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/aws/storage.tf b/aws/storage.tf index 52c804f..d4239c4 100644 --- a/aws/storage.tf +++ b/aws/storage.tf @@ -1,12 +1,18 @@ # S3 bucket to store install and connectors scripts. resource "aws_s3_bucket" "opencti_bucket" { bucket = var.storage_bucket - acl = "private" +} - # Turn on bucket versioning. We'll be storing the Terraform state in S3 and versioning will help protect against human error. - versioning { - enabled = true - } +resource "aws_s3_bucket_versioning" "opencti_bucket_versioning" { + bucket = aws_s3_bucket.opencti_bucket.id + versioning_configuration { + status = "Enabled" + } +} + +resource "aws_s3_bucket_acl" "opencti_bucket_acl" { + bucket = aws_s3_bucket.opencti_bucket.id + acl = "private" } # S3 IAM (I don't think any of these permissions are being used) @@ -34,14 +40,14 @@ resource "aws_iam_role_policy_attachment" "opencti_s3_attach" { } # OpenCTI installer script -resource "aws_s3_bucket_object" "opencti-install-script" { +resource "aws_s3_object" "opencti-install-script" { bucket = aws_s3_bucket.opencti_bucket.id key = "opencti-installer.sh" source = "../opencti_scripts/installer.sh" } # OpenCTI connectors script -resource "aws_s3_bucket_object" "opencti-connectors-script" { +resource "aws_s3_object" "opencti-connectors-script" { bucket = aws_s3_bucket.opencti_bucket.id key = "opencti-connectors.sh" source = "../opencti_scripts/connectors.sh" From db7b04598dd01e5a42d4ea004bc16da6dc1fc873 Mon Sep 17 00:00:00 2001 From: Coleman Kane Date: Sun, 9 Oct 2022 12:13:29 -0400 Subject: [PATCH 24/24] Use gp3 by default, it is faster and cheaper than gp2 --- aws/vm.tf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/aws/vm.tf b/aws/vm.tf index 3c73ae1..6530dec 100644 --- a/aws/vm.tf +++ b/aws/vm.tf @@ -7,6 +7,9 @@ resource "aws_instance" "opencti_instance" { iam_instance_profile = aws_iam_instance_profile.opencti_profile.name root_block_device { volume_size = var.root_volume_size + volume_type = "gp3" + iops = 3000 + throughput = 125 } subnet_id = var.subnet_id