Add support for ARM64 to lambda-promtail drone build job (#5354)

* Add support for ARM64 to lambda-promtail drone build job * update drone.yml based on tomas' changes Signed-off-by: Callum Styan <[email protected]> * Add entry to CHANGELOG.md * Change Docker image for building Promtail Lambda * reverse previous commit Signed-off-by: Callum Styan <[email protected]> * Add manifest for ECR Public images * Minor change to drone.yml for volume mounts + generate drone.jsonnet again. Signed-off-by: Callum Styan <[email protected]> * temp, test drone changes without having to push to main branch Signed-off-by: Callum Styan <[email protected]> * Fix linting issues Signed-off-by: Callum Styan <[email protected]> * temp, push the lambda-promtail images always so we can test the manifest task Signed-off-by: Callum Styan <[email protected]> * We don't build a base arm image, only arm64 Signed-off-by: Callum Styan <[email protected]> * Add volumes definition to ECR manifest pipeline * Regenerate drone.yml with Tomas' most recent commit Signed-off-by: Callum Styan <[email protected]> * Test some some drone changes. Signed-off-by: Callum Styan <[email protected]> * fix lint error in drone.jsonnet Signed-off-by: Callum Styan <[email protected]> * Drop out one of my more recent test changes. Signed-off-by: Callum Styan <[email protected]> * Regenerate drone config. Signed-off-by: Callum Styan <[email protected]> Co-authored-by: Callum Styan <[email protected]>
grafana · Jun 16, 2022 · 35cb40a · 35cb40a
1 parent d6f50ca
commit 35cb40a
Show file tree

Hide file tree

Showing 4 changed files with 209 additions and 15 deletions.
diff --git a/.drone/docker-manifest-ecr.tmpl b/.drone/docker-manifest-ecr.tmpl
@@ -0,0 +1,21 @@
+image: public.ecr.aws/grafana/{{config.target}}:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}
+tags:
+  - main
+{{#if build.tag}}
+  - latest
+{{/if}}
+{{#if build.tags}}
+{{#each build.tags}}
+  - {{this}}
+{{/each}}
+{{/if}}
+manifests:
+  - image: public.ecr.aws/grafana/{{config.target}}:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-amd64
+    platform:
+      architecture: amd64
+      os: linux
+  - image: public.ecr.aws/grafana/{{config.target}}:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-arm64
+    platform:
+      architecture: arm64
+      os: linux
+      variant: v8
diff --git a/.drone/drone.jsonnet b/.drone/drone.jsonnet
@@ -257,17 +257,9 @@ local promtail(arch) = pipeline('promtail-' + arch) + arch_image(arch) {
   depends_on: ['check'],
 };
 
-local lambda_promtail(tags='') = pipeline('lambda-promtail') {
+local lambda_promtail(arch) = pipeline('lambda-promtail-' + arch) + arch_image(arch) {
   steps+: [
-    {
-      name: 'image-tag',
-      image: 'alpine',
-      commands: [
-        'apk add --no-cache bash git',
-        'git fetch origin --tags',
-        'echo $(./tools/image-tag)-amd64 > .tags',
-      ] + if tags != '' then ['echo ",%s" >> .tags' % tags] else [],
-    },
+    // dry run for everything that is not tag or main
     lambda_promtail_ecr('lambda-promtail') {
       depends_on: ['image-tag'],
       when: condition('exclude').tagMain,
@@ -342,6 +334,58 @@ local manifest(apps) = pipeline('manifest') {
   ],
 };
 
+local manifest_ecr(apps, archs) = pipeline('manifest-ecr') {
+  steps: std.foldl(
+    function(acc, app) acc + [{
+      name: 'manifest-' + app,
+      image: 'plugins/manifest',
+      volumes: [{
+        name: 'dockerconf',
+        path: '/.docker',
+      }],
+      settings: {
+        // the target parameter is abused for the app's name,
+        // as it is unused in spec mode. See docker-manifest-ecr.tmpl
+        target: app,
+        spec: '.drone/docker-manifest-ecr.tmpl',
+        ignore_missing: true,
+      },
+      depends_on: ['clone'] + (
+        // Depend on the previous app, if any.
+        if std.length(acc) > 0
+        then [acc[std.length(acc) - 1].name]
+        else []
+      ),
+    }],
+    apps,
+    [{
+      name: 'ecr-login',
+      image: 'docker:dind',
+      volumes: [{
+        name: 'dockerconf',
+        path: '/root/.docker',
+      }],
+      environment: {
+        AWS_ACCESS_KEY_ID: { from_secret: ecr_key.name },
+        AWS_SECRET_ACCESS_KEY: { from_secret: ecr_secret_key.name },
+      },
+      commands: [
+        'apk add --no-cache aws-cli',
+        'docker login --username AWS --password $(aws ecr-public get-login-password --region us-east-1) public.ecr.aws',
+      ],
+      depends_on: ['clone'],
+    }],
+  ),
+  volumes: [{
+    name: 'dockerconf',
+    temp: {},
+  }],
+  depends_on: [
+    'lambda-promtail-%s' % arch
+    for arch in archs
+  ],
+};
+
 [
   pipeline('loki-build-image') {
     workspace: {
@@ -472,6 +516,7 @@ local manifest(apps) = pipeline('manifest') {
         commands: [
           'apk add --no-cache bash git',
           'git fetch origin --tags',
+          'echo $(./tools/image-tag)',
           'echo $(./tools/image-tag) > .tag',
         ],
         depends_on: ['clone'],
@@ -489,5 +534,21 @@ local manifest(apps) = pipeline('manifest') {
     ],
   },
 ] + [promtail_win()]
-+ [lambda_promtail('main')]
-+ [github_secret, pull_secret, docker_username_secret, docker_password_secret, ecr_key, ecr_secret_key, deploy_configuration]
++ [
+  lambda_promtail(arch)
+  for arch in ['amd64', 'arm64']
+] + [
+  manifest_ecr(['lambda-promtail'], ['amd64', 'arm64']) {
+    trigger: condition('include').tagMain {
+      event: ['push'],
+    },
+  },
+] + [
+  github_secret,
+  pull_secret,
+  docker_username_secret,
+  docker_password_secret,
+  ecr_key,
+  ecr_secret_key,
+  deploy_configuration,
+]
diff --git a/.drone/drone.yml b/.drone/drone.yml
@@ -1024,6 +1024,7 @@ steps:
 - commands:
   - apk add --no-cache bash git
   - git fetch origin --tags
+  - echo $(./tools/image-tag)
   - echo $(./tools/image-tag) > .tag
   depends_on:
   - clone
@@ -1074,13 +1075,15 @@ trigger:
 depends_on:
 - check
 kind: pipeline
-name: lambda-promtail
+name: lambda-promtail-amd64
+platform:
+  arch: amd64
+  os: linux
 steps:
 - commands:
   - apk add --no-cache bash git
   - git fetch origin --tags
   - echo $(./tools/image-tag)-amd64 > .tags
-  - echo ",main" >> .tags
   image: alpine
   name: image-tag
 - depends_on:
@@ -1131,6 +1134,114 @@ trigger:
   - pull_request
   - tag
 ---
+depends_on:
+- check
+kind: pipeline
+name: lambda-promtail-arm64
+platform:
+  arch: arm64
+  os: linux
+steps:
+- commands:
+  - apk add --no-cache bash git
+  - git fetch origin --tags
+  - echo $(./tools/image-tag)-arm64 > .tags
+  image: alpine
+  name: image-tag
+- depends_on:
+  - image-tag
+  image: cstyan/ecr
+  name: build-lambda-promtail-image
+  privileged: true
+  settings:
+    access_key:
+      from_secret: ecr_key
+    dockerfile: tools/lambda-promtail/Dockerfile
+    dry_run: true
+    region: us-east-1
+    registry: public.ecr.aws/grafana
+    repo: public.ecr.aws/grafana/lambda-promtail
+    secret_key:
+      from_secret: ecr_secret_key
+  when:
+    ref:
+      exclude:
+      - refs/heads/main
+      - refs/heads/k???
+      - refs/tags/v*
+- depends_on:
+  - image-tag
+  image: cstyan/ecr
+  name: publish-lambda-promtail-image
+  privileged: true
+  settings:
+    access_key:
+      from_secret: ecr_key
+    dockerfile: tools/lambda-promtail/Dockerfile
+    dry_run: false
+    region: us-east-1
+    registry: public.ecr.aws/grafana
+    repo: public.ecr.aws/grafana/lambda-promtail
+    secret_key:
+      from_secret: ecr_secret_key
+  when:
+    ref:
+      include:
+      - refs/heads/main
+      - refs/heads/k???
+      - refs/tags/v*
+trigger:
+  event:
+  - push
+  - pull_request
+  - tag
+---
+depends_on:
+- lambda-promtail-amd64
+- lambda-promtail-arm64
+kind: pipeline
+name: manifest-ecr
+steps:
+- commands:
+  - apk add --no-cache aws-cli
+  - docker login --username AWS --password $(aws ecr-public get-login-password --region
+    us-east-1) public.ecr.aws
+  depends_on:
+  - clone
+  environment:
+    AWS_ACCESS_KEY_ID:
+      from_secret: ecr_key
+    AWS_SECRET_ACCESS_KEY:
+      from_secret: ecr_secret_key
+  image: docker:dind
+  name: ecr-login
+  volumes:
+  - name: dockerconf
+    path: /root/.docker
+- depends_on:
+  - clone
+  - ecr-login
+  image: plugins/manifest
+  name: manifest-lambda-promtail
+  settings:
+    ignore_missing: true
+    spec: .drone/docker-manifest-ecr.tmpl
+    target: lambda-promtail
+  volumes:
+  - name: dockerconf
+    path: /.docker
+trigger:
+  event:
+  - push
+  ref:
+    include:
+    - refs/heads/main
+    - refs/heads/k???
+    - refs/tags/v*
+volumes:
+- name: dockerconf
+  temp: {}
+---
 get:
   name: pat
   path: infra/data/ci/github/grafanabot
@@ -1174,6 +1285,6 @@ kind: secret
 name: deploy_config
 ---
 kind: signature
-hmac: 81be9fce60976703815abf1fd2f1c9255a142d615b90d4b5278ff4a64fa25e92
+hmac: 8335495d285498f513bb265cf90079175c25f5b6c5717deab7abfae65341726a
 
 ...
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -179,6 +179,7 @@ to include only the most relevant.
 * [5409](https://github.com/grafana/loki/pull/5409) **ldb**: Enable best effort parsing for Syslog messages
 * [5392](https://github.com/grafana/loki/pull/5392) **MichelHollands**: Etcd credentials are parsed as secrets instead of plain text now.
 * [5361](https://github.com/grafana/loki/pull/5361) **ctovena**: Add usage report to grafana.com.
+* [5354](https://github.com/grafana/loki/pull/5354) **tlinhart**: Add support for ARM64 to lambda-promtail drone build job.
 * [5289](https://github.com/grafana/loki/pull/5289) **ctovena**: Fix deduplication bug in queries when mutating labels.
 * [5302](https://github.com/grafana/loki/pull/5302) **MasslessParticle** Update azure blobstore client to use new sdk.
 * [5243](https://github.com/grafana/loki/pull/5290) **ssncferreira**: Update Promtail to support duration string formats.