diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command deleted file mode 100755 index c6c7668c1..000000000 --- a/.buildkite/hooks/pre-command +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -## This script prepares the Vault context and required tooling -## for the release pipelines. -## -## NOTE: *_SECRET or *_TOKEN env variables are masked, hence if you'd like to avoid any -## surprises please use the suffix _SECRET or _TOKEN for those values that contain -## any sensitive data. Buildkite can mask those values automatically - -set -eo pipefail - -echo "--- Prepare vault context :vault:" -INTERNAL_CI_TOKEN=$(vault read -field=token secret/ci/elastic-apm-agent-php/internal-ci-token) -export INTERNAL_CI_TOKEN -INTERNAL_CI_JOB_URL_SECRET=$(vault read -field=job-url secret/ci/elastic-apm-agent-php/internal-ci-token) -export INTERNAL_CI_JOB_URL_SECRET diff --git a/.buildkite/release.yml b/.buildkite/release.yml deleted file mode 100644 index c18ce2822..000000000 --- a/.buildkite/release.yml +++ /dev/null @@ -1,14 +0,0 @@ -agents: - provider: "gcp" - -steps: - - label: "Run the release" - key: "release" - commands: .ci/release.sh - artifact_paths: "release.txt" - -notify: - - slack: - channels: - #- "#apm-agent-php" - - "#on-week-oblt-productivity" diff --git a/.ci/.grenrc.js b/.ci/.grenrc.js deleted file mode 100644 index ae731b3ec..000000000 --- a/.ci/.grenrc.js +++ /dev/null @@ -1,26 +0,0 @@ -module.exports = { - "username": "elastic", - "repo": "apm-agent-php", - "dataSource": "prs", - "ignoreIssuesWith": [ - "automation", - "ci", - "developer only" - ], - "groupBy": { - "Breaking changes": ["breaking"], - "Bug fixes": ["bug", "fix"], - "Features": ["enhancement", "feature", "feat"] - }, - "template": { - issue: function (placeholders) { - return '* ' + placeholders.name + ': {pull}' + placeholders.text.replace("#", "") + '[' + placeholders.text + ']'; - }, - changelogTitle: "", - release: "[[release-notes-{{release}}]]\n=== {{release}}\n{{body}}", - releaseSeparator: "", - group: function (placeholders) { - return '\n[float]\n==== ' + placeholders.heading; - } - } -} diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile deleted file mode 100644 index bbb2ea93d..000000000 --- a/.ci/Jenkinsfile +++ /dev/null @@ -1,666 +0,0 @@ -#!/usr/bin/env groovy - -@Library('apm@current') _ - -LOG_LEVEL_NOT_SET = 'Not set' -LOG_LEVELS = [LOG_LEVEL_NOT_SET, 'Trace', 'Debug', 'Info', 'Warning', 'Error', 'Critical', 'Off'] - -pipeline { - agent { label 'ubuntu-18.04 && immutable' } - environment { - REPO = 'apm-agent-php' - BASE_DIR = "src/go.elastic.co/apm/${env.REPO}" - SLACK_CHANNEL = '#apm-agent-php' - NOTIFY_TO = 'build-apm+apm-agent-php@elastic.co' - ONLY_DOCS = "false" - } - options { - buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30')) - timestamps() - ansiColor('xterm') - disableResume() - durabilityHint('PERFORMANCE_OPTIMIZED') - rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true]) - quietPeriod(10) - } - triggers { - issueCommentTrigger("${obltGitHubComments()}") - } - parameters { - string(name: 'VERSION', defaultValue: '', description: "What's the version to be bumped when doing a release?") - choice(name: 'AGENT_LOG_LEVEL', choices: LOG_LEVELS, description: "Agent's log level") - choice(name: 'TESTS_LOG_LEVEL', choices: LOG_LEVELS, description: "Tests' log level") - booleanParam(name: 'INCLUDE_TESTING', defaultValue: true, description: 'Should the testing stages be included?') - } - stages { - stage('Initializing'){ - options { - skipDefaultCheckout() - timeout(time: 2, unit: 'HOURS') - } - stages { - stage('Checkout') { - steps { - whenTrue(isInternalCI() && isTag()) { - setEnvVar('PRE_RELEASE_STAGE', 'true') - notifyStatus(slackStatus: 'good', subject: "[${env.REPO}] Build for the release tag *${env.TAG_NAME}* has been triggered", body: "Build: (<${env.RUN_DISPLAY_URL}|here>) for further details.") - } - whenTrue(params.VERSION?.trim() ? true : false) { - script { - currentBuild.description = "${currentBuild.description?.trim() ? currentBuild.description : ''} release triggered." - } - } - pipelineManager([ cancelPreviousRunningBuilds: [ when: 'PR' ] ]) - deleteDir() - gitCheckout(basedir: "${BASE_DIR}", githubNotifyFirstTimeContributor: true) - stash allowEmpty: true, name: 'source', useDefaultExcludes: false - dir("${BASE_DIR}"){ - // Skip all the stages except docs for PR's with asciidoc and md changes only - whenTrue(isPR()) { - setEnvVar('ONLY_DOCS', isGitRegionMatch(patterns: [ '.*\\.(asciidoc|md|png)' ], shouldMatchAll: true)) - } - } - } - } - stage('Prepare for release (set version, etc.)') { - options { - skipDefaultCheckout() - } - when { - beforeAgent true - // If not a PR and not running in the internalCI and a release with x.y format - allOf { - not { changeRequest() } - not { expression { isInternalCI() } } - expression { return (params.VERSION?.trim() && params.VERSION =~ /^\d+.\d+/) } - } - } - steps { - initWorkspace(context: 'Prepare-Release') { - withGitRelease(credentialsId: '2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken') { - prepareRelease() - } - } - } - } - stage('Build binaries') { - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { return env.ONLY_DOCS == "false" } - } - failFast false - matrix { - agent { label 'ubuntu-18.04 && immutable' } - axes { - axis { - name 'BUILD_ARCHITECTURE' - values 'linux-x86-64', 'linuxmusl-x86-64' - } - } - - stages { - stage('Build PHP extension') { - steps { - initWorkspace(context: "Build-${BUILD_ARCHITECTURE}") { - sh script: "BUILD_ARCHITECTURE=${BUILD_ARCHITECTURE} make -f .ci/Makefile build", label: 'build' - } - withGithubNotify(context: "Build-${BUILD_ARCHITECTURE}") { - dir("${BASE_DIR}"){ - stash includes: "agent/native/_build/${BUILD_ARCHITECTURE}-release/ext/elastic_apm-*", name: "built-extensions-${BUILD_ARCHITECTURE}" - } - } - } - } - } - } - } - stage('PHP tests of extension') { - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { return env.ONLY_DOCS == "false" } - } - failFast false - matrix { - agent { label 'ubuntu-18.04 && immutable' } - axes { - axis { - name 'BUILD_ARCHITECTURE' - values 'linux-x86-64', 'linuxmusl-x86-64' - } - axis { - name 'PHP_VERSION' - // Make sure list of PHP versions supported by the Elastic APM PHP Agent is in sync. - // See the comment in .ci/shared.sh - values '7.2', '7.3', '7.4', '8.0', '8.1', '8.2' - } - } - - stages { - stage('Execute phpt tests') { - when { - beforeAgent true - expression { return params.INCLUDE_TESTING } - } - steps { - initWorkspace(context: "PHPT-${PHP_VERSION}", tab: "tests") { - // When running in the CI with multiple parallel stages - // the access could be considered as a DDOS attack. - retryWithSleep(retries: 3, seconds: 45, backoff: true) { - sh script: "BUILD_ARCHITECTURE=${BUILD_ARCHITECTURE} PHP_VERSION=${PHP_VERSION} make -f .ci/Makefile prepare", label: 'prepare docker image' - } - unstash "built-extensions-${BUILD_ARCHITECTURE}" - sh script: "BUILD_ARCHITECTURE=${BUILD_ARCHITECTURE} PHP_VERSION=${PHP_VERSION} make -f .ci/Makefile run-phpt-tests", label: 'run-phpt-tests' - } - } - } - } - } - } - stage('Static analysis and tests') { - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { return env.ONLY_DOCS == "false" } - } - failFast false - matrix { - agent { label 'ubuntu-18.04 && immutable' } - axes { - axis { - name 'PHP_VERSION' - // Make sure list of PHP versions supported by the Elastic APM PHP Agent is in sync. - // See the comment in .ci/shared.sh - values '7.2', '7.3', '7.4', '8.0', '8.1', '8.2' - } - axis { - name 'DOCKERFILE' - values 'Dockerfile', 'Dockerfile.alpine' - } - } - stages { - stage('Static analysis and unit tests') { - when { - beforeAgent true - expression { return params.INCLUDE_TESTING } - } - steps { - echo "params.INCLUDE_TESTING: ${params.INCLUDE_TESTING}" - echo "params.AGENT_LOG_LEVEL: ${params.AGENT_LOG_LEVEL}" - echo "params.AGENT_LOG_LEVEL == null: " + (params.AGENT_LOG_LEVEL == null) - echo "params.TESTS_LOG_LEVEL: ${params.TESTS_LOG_LEVEL}" - echo "params.TESTS_LOG_LEVEL == null: " + (params.TESTS_LOG_LEVEL == null) - echo "addEnvVarsFromParams([]): " + addEnvVarsFromParams([]) - - initWorkspace(context: "Static-Check-Unit-Tests-${PHP_VERSION}", tab: "tests") { - withEnv(addEnvVarsFromParams([])) { - echo "env.ELASTIC_APM_LOG_LEVEL: ${env.ELASTIC_APM_LOG_LEVEL}" - echo "env.TESTS_LOG_LEVEL: ${env.TESTS_LOG_LEVEL}" - - retryWithSleep(retries: 3, seconds: 45, backoff: true) { - sh script: "PHP_VERSION=${PHP_VERSION} DOCKERFILE=${DOCKERFILE} make -f .ci/Makefile prepare", label: 'prepare docker image' - } - - sh script: "PHP_VERSION=${PHP_VERSION} DOCKERFILE=${DOCKERFILE} make -f .ci/Makefile static-check-unit-test", label: 'static-check-unit-test' - } - } - } - post { - always { - junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/build/*junit.xml") - } - } - } - } - } - } - stage('Build packages') { - when { - beforeAgent true - expression { return env.ONLY_DOCS == "false" } - } - options { skipDefaultCheckout() } - steps { - initWorkspace(context: "Package", tab: 'artifacts') { - // Make sure list of PHP versions supported by the Elastic APM PHP Agent is in sync. - // See the comment in .ci/shared.sh - packageGeneration(versions: ['7.2', '7.3', '7.4', '8.0', '8.1', '8.2']) - } - } - post { - always { - dir("${BASE_DIR}") { - archiveArtifacts(allowEmptyArchive: true, artifacts: 'build/packages/*') - } - } - } - } - stage('Test packages') { - options { skipDefaultCheckout() } - when { - beforeAgent true - allOf { - expression { return env.ONLY_DOCS == "false" } - expression { return params.INCLUDE_TESTING } - not { expression { isInternalCI() } } - } - } - steps { - generatePackageLifecycleTestMatrix(shouldUseSignedBinaries: false) - } - } - } - } - // This meta-stage happens in the internal-ci instance to be able to sign the artifacts correctly. - stage('Release (umbrella stage) (internal-ci)') { - options { - skipDefaultCheckout() - timeout(time: 12, unit: 'HOURS') - } - when { - beforeAgent true - allOf { - tag pattern: 'v\\d+.*', comparator: 'REGEXP' - expression { isInternalCI() } - } - } - agent { label 'ubuntu-18.04 && immutable' } - environment { - BUCKET_NAME = 'internal-ci-artifacts' - BUCKET_SUBFOLDER = "${env.REPO}/${env.TAG_NAME}" - BUCKET_PATH = "gs://${env.BUCKET_NAME}/${env.BUCKET_SUBFOLDER}" - BUCKET_CREDENTIALS = 'internal-ci-gcs-plugin' - SIGNED_ARTIFACTS = 'signed-artifacts' - BUCKET_SUBFOLDER_SIGNED_ARTIFACTS = "${env.BUCKET_SUBFOLDER}/${env.SIGNED_ARTIFACTS}" - BUCKET_SIGNED_ARTIFACTS_PATH = "gs://${env.BUCKET_NAME}/${env.BUCKET_SUBFOLDER_SIGNED_ARTIFACTS}" - RELEASE_URL_MESSAGE = "()" - } - stages { - stage('Sign packages') { - options { skipDefaultCheckout() } - steps { - deleteDir() - unstash 'source' - dir("${BASE_DIR}") { - unstash 'package' - googleStorageUpload(bucket: env.BUCKET_PATH, - credentialsId: env.BUCKET_CREDENTIALS, - pathPrefix: 'build/packages/', - pattern: 'build/packages/**/*', - sharedPublicly: false, - showInline: true) - build(wait: true, propagate: true, job: 'elastic+unified-release+master+sign-artifacts-with-gpg', parameters: [string(name: 'gcs_input_path', value: "${env.BUCKET_PATH}")]) - dir("${SIGNED_ARTIFACTS}") { - googleStorageDownload(bucketUri: "${env.BUCKET_SIGNED_ARTIFACTS_PATH}/*", - credentialsId: env.BUCKET_CREDENTIALS, - localDirectory: 'build/packages/', - pathPrefix: "${env.BUCKET_SUBFOLDER_SIGNED_ARTIFACTS}") - stash allowEmpty: false, name: env.SIGNED_ARTIFACTS, useDefaultExcludes: false - } - archiveArtifacts(allowEmptyArchive: true, artifacts: "${SIGNED_ARTIFACTS}/**/*") - } - } - } - stage('Test signed packages') { - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { return params.INCLUDE_TESTING } - } - steps { - generatePackageLifecycleTestMatrix(shouldUseSignedBinaries: true) - } - post { - unsuccessful { - notifyStatus(slackStatus: 'warning', subject: "[${env.REPO}] Release *${env.TAG_NAME}* got some test failures in the installers.", body: "Please review the signed binaries are healthy (<${env.RUN_DISPLAY_URL}|Open>)") - } - } - } - stage('Ask for approval to publish') { - options { skipDefaultCheckout() } - steps { - setEnvVar('PRE_RELEASE_STAGE', 'false') - notifyStatus(slackStatus: 'warning', subject: "[${env.REPO}@${env.TAG_NAME}] Release ready to be published", - body: "Please (<${env.BUILD_URL}input|approve>) it or reject within 12 hours.\n Changes: ${env.TAG_NAME}") - setEnvVar('HAS_APPROVAL_TO_PUBLISH_RELEASE', prompt(message: "You are about to release version ${env.TAG_NAME}. Do you wish to release it?")) - } - } - stage('Publish as draft release') { - when { - beforeAgent true - expression { return env.HAS_APPROVAL_TO_PUBLISH_RELEASE == 'true' } - } - options { skipDefaultCheckout() } - steps { - deleteDir() - unstash 'source' - dir("${BASE_DIR}") { - unstash "${env.SIGNED_ARTIFACTS}" - withGhEnv(version: '2.4.0') { - sh(script: 'make -f .ci/Makefile draft-release', label: 'draft-release') - setEnvVar('RELEASE_ID', sh(script: 'make -f .ci/Makefile get-draft-release', label: 'get-draft-release', returnStdout: true)?.trim()) - echo "RELEASE_ID: '${env.RELEASE_ID}'" - } - } - } - post { - success { - notifyStatus(slackStatus: 'good', subject: "[${env.REPO}] Draft Release *${env.TAG_NAME}* has been created", body: "Build: (<${env.RUN_DISPLAY_URL}|here>)\nRelease URL: ${env.RELEASE_URL_MESSAGE}") - } - } - } - stage('Verify draft release checksum') { - when { - beforeAgent true - expression { return env.HAS_APPROVAL_TO_PUBLISH_RELEASE == 'true' } - } - options { skipDefaultCheckout() } - steps { - dir("${BASE_DIR}") { - withGhEnv(version: '2.4.0') { - sh(script: "TAG_NAME=${TAG_NAME} ORIGINAL_PACKAGES_LOCATION=build/packages make -f .ci/Makefile download-verify", label: 'download-verify') - } - } - } - post { - always { - archiveArtifacts(allowEmptyArchive: true, artifacts: "${BASE_DIR}/build/packages/target.sha512, ${BASE_DIR}/github/github.sha512") - } - unsuccessful { - notifyStatus( - slackStatus: 'warning', - subject: "[${env.REPO}] published draft release *${env.TAG_NAME}*: artifacts failed checksum verification.", - body: "Please verify if the published draft release binaries are healthy." - + "\nIf the published draft release binaries are invalid please delete the published draft release." - + "\nBuild: (<${env.RUN_DISPLAY_URL}|here>)\nRelease URL: ${env.RELEASE_URL_MESSAGE}" - ) - } - } - } - stage('Publish release (draft -> regular)') { - when { - beforeAgent true - expression { return env.HAS_APPROVAL_TO_PUBLISH_RELEASE == 'true' } - } - options { skipDefaultCheckout() } - steps { - withGhEnv(version: '2.4.0') { - githubReleasePublish(id: "${env.RELEASE_ID}", name: "${env.TAG_NAME}") - } - } - post { - success { - notifyStatus(slackStatus: 'good', subject: "[${env.REPO}] Release *${env.TAG_NAME}* has been published", body: "Build: (<${env.RUN_DISPLAY_URL}|here>)\nRelease URL: ${env.RELEASE_URL_MESSAGE}") - } - } - } - } - post { - failure { - notifyStatus(slackStatus: 'danger', subject: "[${env.REPO}] Release *${env.TAG_NAME}* failed", body: "Build: (<${env.RUN_DISPLAY_URL}|here>)") - } - success { - script { - currentBuild.description = "${currentBuild.description?.trim() ? currentBuild.description : ''} released" - } - } - } - } - } - post { - cleanup { - // Reporting disables in the `internal-ci` since credentials are not in place - // OTOH it avoids duplicated notifications - whenFalse(isInternalCI()){ - notifyBuildResult(prComment: true, analyzeFlakey: true, jobName: getFlakyJobName(withBranch: 'main')) - } - } - failure { - whenTrue(isInternalCI() && env.PRE_RELEASE_STAGE == 'true') { - notifyStatus(slackStatus: 'danger', subject: "[${env.REPO}] Pre-release steps failed", body: "(<${env.RUN_DISPLAY_URL}|Open>)") - } - } - } -} - -/** -* A function to simplify the pipeline and the too large issue in groovy -*/ -def packageGeneration(def args = [:]) { - def local = args.get('local', false) - unstash "built-extensions-linux-x86-64" - unstash "built-extensions-linuxmusl-x86-64" - - if (local) { - // VERSION=1.0.0 is needed to override the current version. - // current version is >1.0.0, and this is the way we can - // run the agent upgrade testing with some local cached - // agent distributions, this one and the current version one. - sh script: "VERSION=1.0.0 make -C packaging package", label: 'package' - sh script: "mv build/packages build/local", label: 'prepare-local-upgrade-agent' - } else { - // Archive the so files to be downloaded if possible. - archiveArtifacts(allowEmptyArchive: true, artifacts: 'agent/native/_build/*-release/ext/elastic_apm-*') - - sh script: "make -C packaging package", label: 'package' - sh script: "make -C packaging info", label: 'package info' - // checksum files are regenerated by the signing component in the internal-ci instance. - stash(includes: 'build/packages/*', name: 'package', excludes: 'build/packages/**/*.sha512') - } -} - -/** -* Init function to -* Send a GitHub Notification, clean the workspace. restore the source code and run -* the body in the BASE_DIR folder -*/ -def initWorkspace(def args = [:], Closure body){ - prepareWorkspace(args) { - dir("${BASE_DIR}"){ - body() - } - } -} - -/** -* Package function to -* Send a GitHub Notification, clean the workspace and restore the source, the package code -* and run the body in the BASE_DIR folder -*/ -def packageWorkspace(def args = [:], Closure body){ - prepareWorkspace(args) { - dir("${BASE_DIR}"){ - unstash (args.shouldUseSignedBinaries ? env.SIGNED_ARTIFACTS : 'package') - // When running in the CI sometimes the docker build might fail for - // some environmental issues, let's retry - retryWithSleep(retries: 3, seconds: 45, backoff: true) { - if (args.prepareGoal != null) { - sh script: "PHP_VERSION=${PHP_VERSION} make -C packaging ${args.prepareGoal}", label: "${args.prepareGoal} for ${PHP_VERSION}" - } - } - body() - } - } -} - -/** -* Prepare function to -* Send a GitHub Notification, clean the workspace and restore the source code. -*/ -def prepareWorkspace(def args = [:], Closure body){ - withGithubNotify(args) { - deleteDir() - unstash 'source' - body() - } -} - -def notifyStatus(def args = [:]) { - releaseNotification(slackChannel: "${env.SLACK_CHANNEL}", - slackColor: args.slackStatus, - slackCredentialsId: 'jenkins-slack-integration-token', - to: "${env.NOTIFY_TO}", - subject: args.subject, - body: args.body) -} - -def prepareRelease() { - def tagName = "v${params.VERSION}" - def branchName = "prepare-release-${tagName}-${env.BUILD_NUMBER}" - def message = "[${params.VERSION}] Bump version and create changelog" - def warning = """> THIS PR IS AUTOMATICALLY GENERATED BY RELEASE PIPELINE.""" - def actions = """### Actions -* If changes are fine then: - 1. [Approve](${env.BUILD_URL}input) it. The pipeline will take care of the release. - 1. Close this PR since it is not required anymore. - 1. Delete branch ${branchName}. -* If changes are not correct then: - 1. [abort](${env.BUILD_URL}input) it. - 1. Change and commit changes to this PR. - 1. Merge this PR. - 1. Delete branch ${branchName}. - 1. Run release [build](${env.JOB_URL}) with the VERSION parameter = ${params.VERSION}""" - def previousVersion = sh(label: 'Get previous version', script: 'make -f .ci/Makefile previous-version', returnStdout: true).trim() - sh label: 'Bump version', script: "VERSION=${params.VERSION} BRANCH_NAME=${env.BRANCH_NAME} make -f .ci/Makefile bump-version" - if (sh(label: 'is version bumped?', script: 'git diff-index --quiet HEAD --', returnStatus: true) > 0) { - // Change ci-tag pointer to be used for generating the changelog - gitCreateTag(tag: 'ci-tag', tagArgs: '--force', pushArgs: '--force') - // Create Changelog and verify it contains what's expected. - sh(label: 'Create changelog', script: "GITHUB_TOKEN=${GITHUB_TOKEN} TAG_NAME=${tagName} PREVIOUS_TAG=v${previousVersion} make -f .ci/Makefile prepare-changelog changelog") - sh(label: "Git branch ${branchName}", script: """git checkout -b ${branchName}""") - sh(label: 'Git commit', script: """git commit -a -m "${message}" """) - def pr = githubCreatePullRequest(title: "[RELEASE] ${params.VERSION}", - description: "${warning}\n\n${actions}", - labels: 'docs,release,changelog', - base: 'main') - notifyStatus(slackStatus: 'warning', subject: "[${env.REPO}] Prepare ${params.VERSION} release steps to be validated.", - body: """Please (<${env.BUILD_URL}input|approve>) it or reject within 12 hours ONLY If no changes are required. Otherwise stop it and review the (<${pr}|PR>).""") - if (prompt(message: "You are about to release version ${params.VERSION}. If you approve then changes will be committed and pushed. Review ${pr}.")) { - // Update branch. - sh(label: "Git branch ${env.BRANCH_NAME}", script: """git checkout ${env.BRANCH_NAME}""") - sh(label: 'Git rebase', script: """git rebase ${branchName}""") - gitPush(args: "${env.BRANCH_NAME}") - gitCreateTag(tag: "${tagName}") - } else { - log(level: 'WARN', text: "Please review the PR ${pr}") - } - } else { - gitCreateTag(tag: "${tagName}") - log(level: 'INFO', text: "There are no changes to compare with. Release will happen from this commit.") - } -} - -def generatePackageLifecycleTestMatrix(def args = [:]) { - def parallelTasks = [:] - dir("${BASE_DIR}") { - def generateScript = ".ci/generate_package_lifecycle_test_matrix.sh" - def output = sh(label: generateScript, script: generateScript, returnStdout: true) - if (output?.trim()) { - output.split('\n').each { testMatrixRow -> - parallelTasks[testMatrixRow] = generatePackageLifecycleTestMatrixRow(testMatrixRow: testMatrixRow, shouldUseSignedBinaries: args.shouldUseSignedBinaries) - } - } else { - error("${generateScript} returned empty set of data") - } - } - parallel(parallelTasks) -} - -def generatePackageLifecycleTestMatrixRow(def args = [:]){ - return { - withNode(labels: 'ubuntu-18.04 && immutable') { - def testingArgs = [:] - testingArgs['shouldUseSignedBinaries'] = args.shouldUseSignedBinaries - testingArgs['testMatrixRow'] = args.testMatrixRow - def parsedTestMatrixRow = args.testMatrixRow.split(',') - testingArgs['phpVersion'] = parsedTestMatrixRow[0] - testingArgs['linuxPackageType'] = parsedTestMatrixRow[1] - def testingType = parsedTestMatrixRow[2] - if (testingType.equals('lifecycle')) { - lifecycleTesting(testingArgs) - } - if (testingType.equals('lifecycle-apache')) { - lifecycleTestingOnProdServerKind(testingArgs + [prodServerKind: 'apache']) - } - if (testingType.equals('lifecycle-fpm')) { - lifecycleTestingOnProdServerKind(testingArgs + [prodServerKind: 'fpm']) - } - if (testingType.equals('php-upgrade')) { - phpUpgradeTesting(testingArgs) - } - if (testingType.equals('agent-upgrade')) { - agentUpgradeTesting(testingArgs) - } - } - } -} - -def addEnvVarsFromParams(def withEnvList) { - if (params.AGENT_LOG_LEVEL != null && !params.AGENT_LOG_LEVEL.equals(LOG_LEVEL_NOT_SET)) { - withEnvList.add('ELASTIC_APM_LOG_LEVEL=' + params.AGENT_LOG_LEVEL) - } - if (params.TESTS_LOG_LEVEL != null && !params.TESTS_LOG_LEVEL.equals(LOG_LEVEL_NOT_SET)) { - withEnvList.add('ELASTIC_APM_PHP_TESTS_LOG_LEVEL=' + params.TESTS_LOG_LEVEL) - } - return withEnvList -} - -def runTestingCommand(def args = [:]) { - try { - args['context'] = buildWorkspaceContext(args) - withEnv(addEnvVarsFromParams(["PHP_VERSION=${args.phpVersion}", "ELASTIC_APM_PHP_TESTS_MATRIX_ROW=${args.testMatrixRow}"])) { - packageWorkspace(args) { - sh(script: args.testingCommand, label: buildLabel(args)) - } - } - } finally { - junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/build/*junit.xml") - } -} - -def buildWorkspaceContext(def args = [:]) { - return args.testMatrixRow + (args.shouldUseSignedBinaries ? "-signed-binaries" : "") -} - -def buildLabel(def args = [:]) { - return "Testing for ${args.testMatrixRow}, shouldUseSignedBinaries: ${args.shouldUseSignedBinaries}" -} - -def lifecycleTesting(def args = [:]) { - runTestingCommand( - args + [ - testingCommand: "make -C packaging ${args.linuxPackageType}-lifecycle-testing" - ] - ) -} - -def lifecycleTestingOnProdServerKind(def args = [:]) { - runTestingCommand( - args + [ - testingCommand: "make -C packaging ${args.linuxPackageType}-lifecycle-testing-in-${args.prodServerKind}", - ] - ) -} - -def phpUpgradeTesting(def args = [:]) { - runTestingCommand( - args + [ - testingCommand: "PHP_VERSION=${args.phpVersion} make -C packaging ${args.linuxPackageType}-php-upgrade-testing" - ] - ) -} - -def agentUpgradeTesting(def args = [:]) { - withEnv(addEnvVarsFromParams(["PHP_VERSION=${args.phpVersion}", "ELASTIC_APM_PHP_TESTS_MATRIX_ROW=${args.testMatrixRow}"])) { - initWorkspace(context: buildWorkspaceContext(args + [prefix: 'Agent-upgrade-testing'])) { - packageGeneration(versions: [args.phpVersion], local: true) - if (args.shouldUseSignedBinaries) { - unstash env.SIGNED_ARTIFACTS - } else { - unstash 'package' - } - sh(script: "make -C packaging ${args.linuxPackageType}-agent-upgrade-testing-local", label: buildLabel(args + [prefix: 'Agent upgrade testing'])) - } - } -} diff --git a/.ci/Makefile b/.ci/Makefile index 6e7aefce5..46f8bb586 100644 --- a/.ci/Makefile +++ b/.ci/Makefile @@ -103,46 +103,6 @@ component-test: prepare ## Run component-test loop: ## Bump the version given VERSION .ci/loop.sh "$(LOOPS)" "${DOCKERFILE}" "${PHP_VERSION}" -.PHONY: previous-version -previous-version: ## Fetch the current version - @grep '\(VERSION = \)' src/ElasticApm/ElasticApm.php | \ - sed "s#.*= '\(.*\)';#\1#g" - -.PHONY: bump-version -bump-version: ## Bump the version given VERSION -ifndef VERSION - @echo "Please set VERSION in the environment to bump the version" - exit 1 -endif - .ci/bump_version.sh - -.PHONY: prepare-changelog -prepare-changelog: ## Prepare the dependencies to run the changelog - @docker build --tag gren -f .ci/docker/gren/Dockerfile . - -.PHONY: changelog -changelog: ## Create the changelog for the given version -ifndef GITHUB_TOKEN - @echo "Please set GITHUB_TOKEN in the environment to generate the changelog" - exit 1 -endif -ifndef TAG_NAME - @echo "Please set TAG_NAME in the environment to create the changelog" - exit 1 -endif -ifndef PREVIOUS_TAG - @echo "Please set PREVIOUS_TAG in the environment to create the changelog" - exit 1 -endif - docker run --rm -t \ - --volume "$(PWD)":/app \ - --workdir /app \ - --env GITHUB_TOKEN=$(GITHUB_TOKEN) \ - --env TAG_NAME=$(TAG_NAME) \ - --env PREVIOUS_TAG=$(PREVIOUS_TAG) \ - -u $(CURRENT_UID):$(CURRENT_GID) \ - gren - .PHONY: draft-release draft-release: validate-tag-name validate-github-token ## Run a draft release given the GITHUB_TOKEN and TAG_NAME gh \ @@ -159,6 +119,14 @@ draft-release: validate-tag-name validate-github-token ## Run a draft release g get-draft-release: validate-tag-name ## Get the draft release id for the given TAG_NAME @gh api repos/{owner}/{repo}/releases | jq --arg v "$(TAG_NAME)" '.[]|select(.draft)|select(.tag_name == $$v)|.id' +.PHONY: github-release-ready +github-release-ready: validate-tag-name validate-github-token ## Transition the draft release to ready given the GITHUB_TOKEN and TAG_NAME + gh \ + release \ + edit \ + "$(TAG_NAME)" \ + --draft=false + .PHONY: download-verify download-verify: ## Download the assets for the given draft release and verify their signature .ci/download_release_and_verify_checksum.sh "${TAG_NAME}" "${ORIGINAL_PACKAGES_LOCATION}" diff --git a/.ci/bump_version.sh b/.ci/bump_version.sh deleted file mode 100755 index d432cbe19..000000000 --- a/.ci/bump_version.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -set -xe - -sed -i.bck "s#\(VERSION = \).*;#\1'${VERSION}';#g" agent/php/ElasticApm/ElasticApm.php -sed -i.bck "s#\(PHP_ELASTIC_APM_VERSION\).*#\1 \"${VERSION}\"#g" agent/native/ext/elastic_apm_version.h - -git add agent/php/ElasticApm/ElasticApm.php agent/native/ext/elastic_apm_version.h diff --git a/.ci/jobs/apm-agent-php.yml b/.ci/jobs/apm-agent-php.yml deleted file mode 100644 index 9172dc193..000000000 --- a/.ci/jobs/apm-agent-php.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -- job: - name: apm-agent-php - description: apm-agent-php - project-type: folder diff --git a/.ci/jobs/defaults.yml b/.ci/jobs/defaults.yml deleted file mode 100644 index 345c49ab3..000000000 --- a/.ci/jobs/defaults.yml +++ /dev/null @@ -1,57 +0,0 @@ ---- - -##### GLOBAL METADATA - -- meta: - cluster: apm-ci - -##### JOB DEFAULTS - -- job: - view: APM-CI - project-type: multibranch - logrotate: - numToKeep: 100 - concurrent: true - node: linux - script-path: .ci/Jenkinsfile - scm: - - github: - branch-discovery: no-pr - discover-pr-forks-strategy: merge-current - discover-pr-forks-trust: permission - discover-pr-origin: merge-current - discover-tags: true - notification-context: 'apm-ci' - repo: apm-agent-php - repo-owner: elastic - credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken - ssh-checkout: - credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba - build-strategies: - - tags: - ignore-tags-older-than: -1 - ignore-tags-newer-than: -1 - - regular-branches: true - - change-request: - ignore-target-only-changes: false - clean: - after: true - before: true - prune: true - shallow-clone: true - depth: 3 - do-not-fetch-tags: true - submodule: - disable: false - recursive: true - parent-credentials: true - timeout: 100 - timeout: '15' - use-author: true - wipe-workspace: 'True' - periodic-folder-trigger: 1w - prune-dead-branches: true - publishers: - - email: - recipients: infra-root+build@elastic.co diff --git a/.github/workflows/README.md b/.github/workflows/README.md index a0580a09e..8e9bf3b10 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -33,7 +33,7 @@ Every time there is a merge to main or any release branches the whole workflow w ### Release process -This process has been fully automated and it gets triggered when a tag release has been created. +This process has been fully automated and [it](https://github.com/elastic/apm-agent-php/actions/workflows/release.yml) gets triggered when a tag release has been created. The tag release follows the naming convention: `v...`, where ``, `` and ``. ### OpenTelemetry diff --git a/.github/workflows/generate-matrix.yml b/.github/workflows/generate-matrix.yml new file mode 100644 index 000000000..ba0f4e0c7 --- /dev/null +++ b/.github/workflows/generate-matrix.yml @@ -0,0 +1,24 @@ +--- + +# Runs the generate matrix based on the provided files in test.yml or release.yml +name: generate-matrix + +on: + workflow_call: + outputs: + include: + description: "The matrix" + value: ${{ jobs.generate-matrix.outputs.include }} + +jobs: + generate-matrix: + timeout-minutes: 5 + runs-on: ubuntu-latest + outputs: + include: ${{ steps.generate.outputs.matrix }} + steps: + - uses: actions/checkout@v3 + - id: generate + run: | + MATRIX=$(.ci/generate_package_lifecycle_test_matrix.sh | jq --raw-input --slurp -c 'split("\n") | map(select(length > 0)) | map(split(",")) | map({ "item": . } )') + echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/opentelemetry.yml b/.github/workflows/opentelemetry.yml index f80c597c1..bde2807a4 100644 --- a/.github/workflows/opentelemetry.yml +++ b/.github/workflows/opentelemetry.yml @@ -10,6 +10,7 @@ on: - updatecli - "Issue Labeler" - "Auto Assign to Project(s)" + - release - snapshoty types: [completed] diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0c6b7113e..05aa719b0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,115 +5,141 @@ permissions: contents: read on: - workflow_dispatch: - ### TODO: listen for tags, so for now let's test whether it works in dry run mode by default - #create: - # tags: [ "v*" ] + create: + tags: [ "v*" ] jobs: + build: - name: build-agent-library - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - arch: - - "linux-x86-64" - - "linuxmusl-x86-64" - env: - BUILD_ARCHITECTURE: ${{ matrix.arch }} - steps: - - uses: actions/checkout@v3 - - name: Build - run: make -f .ci/Makefile build - - uses: actions/upload-artifact@v3 - with: - name: package-parts-${{ matrix.arch }} - path: | - agent/native/_build/${{ matrix.arch }}-release/ext/elastic_apm*.so - agent/native/_build/${{ matrix.arch }}-release/ext/elastic_apm*.debug + uses: ./.github/workflows/build.yml + build-packages: - runs-on: ubuntu-latest needs: - build - steps: - - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 - with: - name: package-parts-linux-x86-64 - path: agent/native/_build/linux-x86-64-release/ext - - uses: actions/download-artifact@v3 - with: - name: package-parts-linuxmusl-x86-64 - path: agent/native/_build/linuxmusl-x86-64-release/ext - - name: package - run: make -C packaging package - - name: package info - run: make -C packaging info - - uses: actions/upload-artifact@v3 - with: - name: package - path: | - build/packages/* - !build/packages/**/*.sha512 + uses: ./.github/workflows/build-packages.yml - release: - name: Release + sign: runs-on: ubuntu-latest - + needs: + - build-packages + env: + BUCKET_NAME: "apm-agent-php" steps: - uses: actions/checkout@v3 - uses: actions/download-artifact@v3 with: name: package path: build/packages - - run: echo "Upload artifacts to gs://internal-ci-artifacts/apm-agent-php/" + + ## NOTE: The name of the zip should match the name of the folder to be zipped. + - name: Prepare packages to be signed + run: |- + cd build + zip -r packages.zip packages/ + + - name: 'Get service account' + uses: hashicorp/vault-action@v2.7.3 + with: + url: ${{ secrets.VAULT_ADDR }} + roleId: ${{ secrets.VAULT_ROLE_ID }} + secretId: ${{ secrets.VAULT_SECRET_ID }} + method: approle + secrets: | + secret/observability-team/ci/apm-agent-php-bucket service-account | SERVICE_ACCOUNT ; + + - name: 'Authenticate to Google Cloud' + uses: 'google-github-actions/auth@v1' + with: + credentials_json: '${{ env.SERVICE_ACCOUNT }}' + + - id: 'upload-file' + uses: 'google-github-actions/upload-cloud-storage@v1' + with: + path: "build/packages.zip" + destination: "${{ env.BUCKET_NAME }}/${{ github.run_id }}" + predefinedAcl: "publicRead" - id: buildkite - name: Run Release + name: Run buildkite pipeline uses: elastic/apm-pipeline-library/.github/actions/buildkite@current with: vaultUrl: ${{ secrets.VAULT_ADDR }} vaultRoleId: ${{ secrets.VAULT_ROLE_ID }} vaultSecretId: ${{ secrets.VAULT_SECRET_ID }} - pipeline: apm-agent-php-release + pipeline: observability-robots-php-release + triggerMessage: "${{ github.repository }}@${{ github.ref_name }} - sign artifacts" waitFor: true printBuildLogs: true + artifactName: signed-artifacts + artifactPath: "signed-artifacts.zip" buildEnvVars: | - dry_run=true + BUNDLE_URL=https://storage.googleapis.com/${{ env.BUCKET_NAME }}/${{ steps.upload-file.outputs.uploaded }} - - run: echo "Download artifacts from gs://internal-ci-artifacts/apm-agent-php//signed-artifacts" + generate-test-packages-matrix: + uses: ./.github/workflows/generate-matrix.yml - - run: echo "Run tests" + test-packages: + needs: + - sign + - generate-test-packages-matrix + uses: ./.github/workflows/test-packages.yml + with: + include: ${{ needs.generate-test-packages-matrix.outputs.include }} + max-parallel: 40 - - run: echo "here is no approval input" + release: + needs: + - test-packages + runs-on: ubuntu-latest + permissions: + contents: write + env: + GITHUB_TOKEN: ${{ github.token }} + TAG_NAME: ${{ github.ref_name }} + steps: - - run: echo "Publish draft release" + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 + with: + name: signed-artifacts + path: build/packages - - run: echo "Verify draft release checksum" + - name: Unzip signed packages + run: |- + cd build/packages + unzip signed-artifacts.zip - - run: echo "Publish release (draft -> regular)" + - name: Create draft release + run: |- + make -f .ci/Makefile draft-release + echo "RELEASE_ID=$(make -f .ci/Makefile get-draft-release)" >> $GITHUB_ENV - - if: ${{ success() }} - uses: elastic/apm-pipeline-library/.github/actions/slack-message@current - with: - url: ${{ secrets.VAULT_ADDR }} - roleId: ${{ secrets.VAULT_ROLE_ID }} - secretId: ${{ secrets.VAULT_SECRET_ID }} - #channel: "#apm-agent-php" - channel: "#on-week-oblt-productivity" - message: | - :runner: [${{ github.repository }}] Release *${{ github.ref_name }}* has been triggered in Buildkite: (<${{ steps.buildkite.outputs.build }}|build>) + - name: Verify draft release + run: ORIGINAL_PACKAGES_LOCATION=build/packagesmake -f .ci/Makefile download-verify - - if: ${{ failure() }} - uses: elastic/apm-pipeline-library/.github/actions/slack-message@current + - name: Publish release + run: make -f .ci/Makefile github-release-ready + + notify: + if: always() + needs: + - build + - build-packages + - generate-test-packages-matrix + - release + - sign + - test-packages + runs-on: ubuntu-latest + steps: + - id: check + uses: elastic/apm-pipeline-library/.github/actions/check-dependent-jobs@current with: - url: ${{ secrets.VAULT_ADDR }} - roleId: ${{ secrets.VAULT_ROLE_ID }} - secretId: ${{ secrets.VAULT_SECRET_ID }} - #channel: "#apm-agent-php" - channel: "#on-week-oblt-productivity" - message: | - :ghost: [${{ github.repository }}] Release *${{ github.ref_name }}* didn't get triggered in Buildkite. - Build: (<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|here>) + needs: ${{ toJSON(needs) }} + - uses: elastic/apm-pipeline-library/.github/actions/notify-build-status@current + with: + status: ${{ steps.check.outputs.status }} + vaultUrl: ${{ secrets.VAULT_ADDR }} + vaultRoleId: ${{ secrets.VAULT_ROLE_ID }} + vaultSecretId: ${{ secrets.VAULT_SECRET_ID }} + slackChannel: "#apm-agent-php" + message: "[${{ github.repository }}] Release (<${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ github.ref_name }}|${{ github.ref_name }}>)" diff --git a/.github/workflows/test-packages.yml b/.github/workflows/test-packages.yml index 4da7967fc..052793458 100644 --- a/.github/workflows/test-packages.yml +++ b/.github/workflows/test-packages.yml @@ -6,16 +6,21 @@ name: test-packages on: workflow_call: inputs: - include: - required: true - type: string + include: + required: true + type: string + max-parallel: + description: 'Set the maximum number of jobs that can run simultaneously in the matrix' + default: 20 + required: false + type: number jobs: - build: + test-packages: timeout-minutes: 120 runs-on: ubuntu-latest strategy: - max-parallel: 20 + max-parallel: ${{ inputs.max-parallel }} fail-fast: false matrix: include: ${{ fromJSON(inputs.include) }} diff --git a/.github/workflows/test-reporter.yml b/.github/workflows/test-reporter.yml index c15b21cdb..5e90d4559 100644 --- a/.github/workflows/test-reporter.yml +++ b/.github/workflows/test-reporter.yml @@ -5,6 +5,7 @@ name: test-reporter on: workflow_run: workflows: + - release - test types: [completed] diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e3a74533a..43518a05b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -107,15 +107,7 @@ jobs: uses: ./.github/workflows/build-packages.yml generate-test-packages-matrix: - runs-on: ubuntu-latest - outputs: - include: ${{ steps.generate.outputs.matrix }} - steps: - - uses: actions/checkout@v3 - - id: generate - run: | - MATRIX=$(.ci/generate_package_lifecycle_test_matrix.sh | jq --raw-input --slurp -c 'split("\n") | map(select(length > 0)) | map(split(",")) | map({ "item": . } )') - echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT + uses: ./.github/workflows/generate-matrix.yml test-packages: needs: