diff --git a/aws/main.tf b/aws/main.tf index b15c520..4b75534 100644 --- a/aws/main.tf +++ b/aws/main.tf @@ -1,6 +1,6 @@ provider "aws" { region = var.region - shared_credentials_file = "~/.aws/credentials" + shared_credentials_files = ["$HOME/.aws/credentials"] profile = "default" } diff --git a/aws/storage.tf b/aws/storage.tf index 52c804f..d4239c4 100644 --- a/aws/storage.tf +++ b/aws/storage.tf @@ -1,12 +1,18 @@ # S3 bucket to store install and connectors scripts. resource "aws_s3_bucket" "opencti_bucket" { bucket = var.storage_bucket - acl = "private" +} - # Turn on bucket versioning. We'll be storing the Terraform state in S3 and versioning will help protect against human error. - versioning { - enabled = true - } +resource "aws_s3_bucket_versioning" "opencti_bucket_versioning" { + bucket = aws_s3_bucket.opencti_bucket.id + versioning_configuration { + status = "Enabled" + } +} + +resource "aws_s3_bucket_acl" "opencti_bucket_acl" { + bucket = aws_s3_bucket.opencti_bucket.id + acl = "private" } # S3 IAM (I don't think any of these permissions are being used) @@ -34,14 +40,14 @@ resource "aws_iam_role_policy_attachment" "opencti_s3_attach" { } # OpenCTI installer script -resource "aws_s3_bucket_object" "opencti-install-script" { +resource "aws_s3_object" "opencti-install-script" { bucket = aws_s3_bucket.opencti_bucket.id key = "opencti-installer.sh" source = "../opencti_scripts/installer.sh" } # OpenCTI connectors script -resource "aws_s3_bucket_object" "opencti-connectors-script" { +resource "aws_s3_object" "opencti-connectors-script" { bucket = aws_s3_bucket.opencti_bucket.id key = "opencti-connectors.sh" source = "../opencti_scripts/connectors.sh" diff --git a/aws/vm.tf b/aws/vm.tf index 3c73ae1..6530dec 100644 --- a/aws/vm.tf +++ b/aws/vm.tf @@ -7,6 +7,9 @@ resource "aws_instance" "opencti_instance" { iam_instance_profile = aws_iam_instance_profile.opencti_profile.name root_block_device { volume_size = var.root_volume_size + volume_type = "gp3" + iops = 3000 + throughput = 125 } subnet_id = var.subnet_id diff --git a/opencti_scripts/connectors.sh b/opencti_scripts/connectors.sh index 130004c..f815fb8 100644 --- a/opencti_scripts/connectors.sh +++ b/opencti_scripts/connectors.sh @@ -120,6 +120,10 @@ if [[ ${ubuntu_version} == 18 ]] then python_ver="3.7" elif [[ ${ubuntu_version} == 20 ]] +then + # Using bionic since focal not avaialble yet for RabbitMQ + python_ver="3" +elif [[ ${ubuntu_version} == 22 ]] then # Using bionic since focal not avaialble yet for RabbitMQ python_ver="3" @@ -140,26 +144,26 @@ warn_user # This will only set up your instance for the connectors enabled. You must supply an API token (e.g., alienvault token) and enable the service. # It should be safe to run this after changing configs or enabling services. declare -A CONNECTORS; -CONNECTORS['alienvault']=0 -CONNECTORS['amitt']=0 -CONNECTORS['crowdstrike']=0 -CONNECTORS['cryptolaemus']=0 -CONNECTORS['cve']=1 -CONNECTORS['cyber-threat-coalition']=0 -CONNECTORS['cybercrime-tracker']=0 -CONNECTORS['export-file-csv']=1 -CONNECTORS['export-file-stix']=1 -CONNECTORS['hygiene']=0 -CONNECTORS['import-file-pdf-observables']=1 -CONNECTORS['import-file-stix']=1 -CONNECTORS['ipinfo']=0 -CONNECTORS['lastinfosec']=0 -CONNECTORS['malpedia']=0 -CONNECTORS['misp']=1 -CONNECTORS['mitre']=1 -CONNECTORS['opencti']=1 -CONNECTORS['valhalla']=0 -CONNECTORS['virustotal']=1 +CONNECTORS['external-import/alienvault']=0 +CONNECTORS['external-import/amitt']=0 +CONNECTORS['external-import/crowdstrike']=0 +CONNECTORS['external-import/cryptolaemus']=0 +CONNECTORS['external-import/cve']=1 +CONNECTORS['external-import/cyber-threat-coalition']=0 +CONNECTORS['external-import/cybercrime-tracker']=0 +CONNECTORS['internal-export-file/export-file-csv']=1 +CONNECTORS['internal-export-file/export-file-stix']=1 +CONNECTORS['internal-enrichment/hygiene']=0 +CONNECTORS['internal-import-file/import-document']=1 +CONNECTORS['internal-import-file/import-file-stix']=1 +CONNECTORS['internal-enrichment/ipinfo']=0 +CONNECTORS['external-import/lastinfosec']=0 +CONNECTORS['external-import/malpedia']=0 +CONNECTORS['external-import/misp']=1 +CONNECTORS['external-import/mitre']=1 +CONNECTORS['external-import/opencti']=1 +CONNECTORS['external-import/valhalla']=1 +CONNECTORS['internal-enrichment/virustotal']=1 echo "The following connectors will be installed:" for i in "${!CONNECTORS[@]}" @@ -200,16 +204,31 @@ do sed -i"" -e "s/id: 'ChangeMe'/id: '$(uuidgen -r | tr -d '\n' | tr '[:upper:]' '[:lower:]')'/g" "${opencti_connector_dir}/$i/src/config.yml" fi - if [[ ! -f "/etc/systemd/system/opencti-connector-$i.service" ]] + sbasename=$(basename "$i") + scriptname="$sbasename" + + # Some of the services use the service name as the main Python script name, while + # others have started using "main.py". This check attempts to discover which the + # service in question is using, and adapts to it, falling back on the old behavior + if [[ -f "${opencti_connector_dir}/$i/src/main.py" ]]; then + scriptname="main" + fi + + if [[ ! -f "/etc/systemd/system/opencti-connector-$sbasename.service" ]] then - cat > /etc/systemd/system/opencti-connector-$i.service <<- EOT + cat > /etc/systemd/system/opencti-connector-$sbasename.service <<- EOT [Unit] Description=OpenCTI Connector - $i After=network.target +StartLimitBurst=30 +StartLimitInterval=0 + [Service] +RestartSec=20 +TimeoutStartSec=600 Type=simple WorkingDirectory=${opencti_connector_dir}/$i/src -ExecStart=/usr/bin/python${python_ver} "${opencti_connector_dir}/$i/src/$i.py" +ExecStart=/usr/bin/python${python_ver} "${opencti_connector_dir}/$i/src/$scriptname.py" ExecReload=/bin/kill -s HUP \$MAINPID ExecStop=/bin/kill -s TERM \$MAINPID PrivateTmp=true @@ -219,16 +238,16 @@ WantedBy=multi-user.target EOT systemctl daemon-reload - systemctl start opencti-connector-$i.service + systemctl start opencti-connector-$sbasename.service fi - if [[ $(systemctl status --no-pager opencti-connector-$i.service | grep 'Active: active') ]] + if [[ $(systemctl status --no-pager opencti-connector-$sbasename.service | grep 'Active: active') ]] then - echo "opencti-connector-$i.service is already running, restarting due to config changes" - systemctl restart opencti-connector-$i.service + echo "opencti-connector-$sbasename.service is already running, restarting due to config changes" + systemctl restart opencti-connector-$sbasename.service fi - quit_on_error "Installing service for connector: $i" + quit_on_error "Installing service for connector: $sbasename" fi done diff --git a/opencti_scripts/installer.sh b/opencti_scripts/installer.sh index a2bbcbb..9b96d37 100644 --- a/opencti_scripts/installer.sh +++ b/opencti_scripts/installer.sh @@ -126,7 +126,7 @@ function check_apt_pkg { } # Function: check_service -# Checks if a service is active or nah. Matches Grakn service output. +# Checks if a service is active or nah. Matches TypeDB service output. # Parameters: # - $1: service to check function check_service { @@ -194,6 +194,11 @@ then distro="bionic" run_python="python3.7" elif [[ ${ubuntu_version} == 20 ]] +then + # Using bionic since focal not avaialble yet for RabbitMQ + distro="bionic" + run_python="python3" +elif [[ ${ubuntu_version} == 22 ]] then # Using bionic since focal not avaialble yet for RabbitMQ distro="bionic" @@ -202,20 +207,17 @@ else quit_on_error echo "You are using an unsupported version of Ubuntu. Exiting." fi -# Grakn -grakn_bin_version="2.0.0-alpha-6" -grakn_console_version="2.0.0-alpha-4" -grakn_core_all_version="2.0.0-alpha-4" -grakn_core_server_version="2.0.0-alpha-4" - -# Minio -minio_dir="/opt/minio/data" +# TypeDB +typedb_bin_version="2.9.0" +typedb_console_version="2.11.1" +typedb_core_all_version="2.11.1" +typedb_core_server_version="2.11.1" # Redis -redis_ver="6.0.5" +redis_ver="7.0.5" # RabbitMQ -rabbitmq_ver="3.8.5-1" +rabbitmq_ver="3.10.5-1" rabbitmq_release_url="https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc" # OpenCTI @@ -232,9 +234,9 @@ do esac done -opencti_ver="4.2.1" +opencti_ver="5.3.17" opencti_dir="/opt/opencti" -opencti_worker_count=2 +opencti_worker_count=4 # ########### # Main script @@ -262,8 +264,7 @@ disable_service 'opencti-server' disable_service 'elasticsearch' disable_service 'redis-server' disable_service 'rabbitmq-server' -disable_service 'minio' -disable_service 'grakn' +disable_service 'typedb' # The VMs we're running are not that big and we're going to quickly fill the system log with our work (and especially the connectors). This will max out the logs at 100M. echo "SystemMaxUse=100M" >> /etc/systemd/journald.conf @@ -304,34 +305,36 @@ check_apt_pkg "python3-pip" ${run_python} -m pip install --upgrade pip ${run_python} -m pip -q install --ignore-installed PyYAML -## Grakn -log_section_heading "Grakn" +## TypeDB +log_section_heading "TypeDB" sudo apt-key adv --keyserver keyserver.ubuntu.com --recv 8F3DA4B5E9AEF44C -sudo add-apt-repository 'deb [ arch=all ] https://repo.grakn.ai/repository/apt/ trusty main' +sudo add-apt-repository 'deb [ arch=all ] https://repo.vaticle.com/repository/apt/ trusty main' update_apt_pkg # apt-get install -y grakn-console=2.0.0-alpha-3 # Required dependency # apt-get install -y grakn-core-all -check_apt_pkg 'grakn-bin' "=${grakn_bin_version}" -check_apt_pkg 'grakn-core-server' "=${grakn_core_server_version}" -check_apt_pkg 'grakn-console' "=${grakn_console_version}" -check_apt_pkg 'grakn-core-all' "=${grakn_core_all_version}" - -### Create systemd unit file for Grakn -cat < /etc/systemd/system/grakn.service +# check_apt_pkg 'grakn-bin' "=${grakn_bin_version}" +# check_apt_pkg 'grakn-core-server' "=${grakn_core_server_version}" +# check_apt_pkg 'grakn-console' "=${grakn_console_version}" +check_apt_pkg 'typedb-bin' "=${typedb_bin_version}" +check_apt_pkg 'typedb-server' "=${typedb_core_all_version}" +check_apt_pkg 'typedb-all' "=${typedb_core_all_version}" + +### Create systemd unit file for TypeDB +cat < /etc/systemd/system/typedb.service [Unit] -Description=Grakn.AI Server daemon +Description=TypeDB Server daemon After=network.target [Service] -Type=forking -ExecStart=/usr/local/bin/grakn server start -ExecStop=/usr/local/bin/grakn server stop -ExecReload=/usr/local/bin/grakn server stop && /usr/local/bin/grakn server start -RemainAfterExit=yes +Type=simple +ExecStart=/usr/local/bin/typedb server +#ExecStop=/usr/local/bin/typedb server stop +#ExecReload=/usr/local/bin/typedb server stop && /usr/local/bin/typedb server start +#RemainAfterExit=yes [Install] WantedBy=multi-user.target EOT systemctl daemon-reload -enable_service 'grakn' +enable_service 'typedb' ## Elasticsearch log_section_heading "Elasticsearch" @@ -351,49 +354,14 @@ cat < /etc/logrotate.d/elasticsearch } EOT wget -qO - 'https://artifacts.elastic.co/GPG-KEY-elasticsearch' | apt-key add - -add-apt-repository "deb https://artifacts.elastic.co/packages/7.x/apt stable main" +add-apt-repository "deb https://artifacts.elastic.co/packages/8.x/apt stable main" update_apt_pkg check_apt_pkg 'elasticsearch' +sed -i 's|xpack.security.enabled: true|xpack.security.enabled: false|' /etc/elasticsearch/elasticsearch.yml +sed -i 's|xpack.security.enrollment.enabled: true|xpack.security.enrollment.enabled: false|' /etc/elasticsearch/elasticsearch.yml +sed -i 's|^ enabled: true| enabled: false|' /etc/elasticsearch/elasticsearch.yml enable_service 'elasticsearch' -## Minio -log_section_heading "Minio" -wget --quiet -O minio https://dl.min.io/server/minio/release/linux-amd64/minio -chmod +x minio -mv minio "/usr/local/bin/" -if [[ ! -d "${minio_dir}" ]] -then - mkdir -p "${minio_dir}" -fi - -### From: https://github.com/minio/minio-service/blob/master/linux-systemd/minio.service -if [[ ! -f "/etc/default/minio" ]] -then - # .minio.access_key - RMINIOAK="$(openssl rand -hex 12)" - # .minio.secret_key - RMINIOSK="$(openssl rand -base64 25 | tr -d '/')" - cat > /etc/default/minio <<- EOT -# Volume to be used for MinIO server. -MINIO_VOLUMES="/opt/minio/data/" -# Use if you want to run MinIO on a custom port. -# MINIO_OPTS="--address :9199" -# Access Key of the server. -MINIO_ACCESS_KEY=${RMINIOAK} -# Secret key of the server. -MINIO_SECRET_KEY=${RMINIOSK} -EOT -else - RMINIOAK="$(grep -o 'MINIO_ACCESS_KEY=.*' /etc/default/minio | cut -f2- -d=)" - RMINIOSK="$(grep -o 'MINIO_SECRET_KEY=.*' /etc/default/minio | cut -f2- -d=)" -fi - -curl "https://raw.githubusercontent.com/minio/minio-service/master/linux-systemd/minio.service" -o "/etc/systemd/system/minio.service" -sed -i'' -e 's/User=minio-user/User=root/g' "/etc/systemd/system/minio.service" -sed -i'' -e 's/Group=minio-user/Group=root/g' "/etc/systemd/system/minio.service" -systemctl daemon-reload -enable_service 'minio' - ## Redis log_section_heading "Redis" update_apt_pkg @@ -436,7 +404,7 @@ fi if [[ ! -f "/etc/redis/redis.conf" ]] then cp "redis-${redis_ver}/redis.conf" "/etc/redis/redis.conf" - sed -i 's/^supervised no/supervised systemd/' "/etc/redis/redis.conf" + sed -i 's/^\#\ supervised\ .*$/supervised auto/' "/etc/redis/redis.conf" chown redis:redis "/etc/redis/redis.conf" fi @@ -457,7 +425,7 @@ Type=notify ExecStart=/usr/local/bin/redis-server /etc/redis/redis.conf ExecStop=/usr/local/bin/redis-cli -p 6379 shutdown ExecReload=/bin/kill -USR2 \$MAINPID -TimeoutStartSec=10 +TimeoutStartSec=900 TimeoutStopSec=10 Restart=on-failure [Install] @@ -471,13 +439,23 @@ enable_service 'redis-server' ## RabbitMQ log_section_heading "RabbitMQ" curl -fsSL "${rabbitmq_release_url}" | apt-key add - -tee /etc/apt/sources.list.d/bintray.rabbitmq.list < ${opencti_dir}/config/production.json +cat ${opencti_dir}/config/default.json | jq ".app.admin.email=\"${opencti_email}\" | .app.admin.password=\"${RADMINPASS}\" | .app.admin.token=\"${RADMINTOKEN}\" | .minio.bucket_name=\"${storage_bucket}\" | .minio.endpoint=\"s3.amazonaws.com\" | .minio.port=443 | .minio.use_ssl=true | .minio.access_key=\"\" | .minio.secret_key=\"\" | .minio.use_aws_role=true | .rabbitmq.username=\"${RRMQUNAME}\" | .rabbitmq.password=\"${RRMQPASS}\"" > ${opencti_dir}/config/production.json echo "OpenCTI: Create unit file" cat > /etc/systemd/system/opencti-server.service <<- EOT @@ -578,7 +555,12 @@ cat > /etc/systemd/system/opencti-worker@.service <<- EOT [Unit] Description=OpenCTI Worker daemon %i After=network.target opencti-server.service +StartLimitBurst=30 +StartLimitInterval=0 + [Service] +RestartSec=20 +TimeoutStartSec=600 Type=simple WorkingDirectory=${opencti_dir}/worker/ ExecStart=/usr/bin/${run_python} "${opencti_dir}/worker/worker.py" diff --git a/userdata/installation-wrapper-script.sh b/userdata/installation-wrapper-script.sh index 3dd2ed9..1a8b8b2 100644 --- a/userdata/installation-wrapper-script.sh +++ b/userdata/installation-wrapper-script.sh @@ -41,10 +41,10 @@ chmod +x /opt/${connectors_script_name} echo "Starting OpenCTI installation script" # Run the install script with the provided e-mail address. # AWS automatically runs the script as root, Azure doesn't. -sudo /opt/${install_script_name} -e "${login_email}" +sudo /usr/bin/env storage_bucket="${storage_bucket}" /opt/${install_script_name} -e "${login_email}" echo "OpenCTI installation script complete." echo "Starting OpenCTI connectors script." # Run the script without prompting the user (the default, `-p 0`, will prompt if the user wants to apply; this is less than ideal for an automated script). -sudo /opt/${connectors_script_name} -p 1 +sudo /usr/bin/env storage_bucket="${storage_bucket}" /opt/${connectors_script_name} -p 1 echo "OpenCTI wrapper script complete."