From 64a226a958cefab9dcf9193d82439f5bd7ca1e12 Mon Sep 17 00:00:00 2001 From: Aidan Oldershaw Date: Mon, 19 Jul 2021 16:48:32 -0400 Subject: [PATCH 1/4] add multi-branch workflow example Signed-off-by: Aidan Oldershaw --- pipelines/multi-branch/template.yml | 81 ++++++++++++++++++++++++++++ pipelines/multi-branch/tracker.yml | 83 +++++++++++++++++++++++++++++ tasks/go-build.yml | 28 ++++++++++ tasks/go-test.yml | 20 +++++++ terraform/staging/main.tf | 13 +++++ 5 files changed, 225 insertions(+) create mode 100644 pipelines/multi-branch/template.yml create mode 100644 pipelines/multi-branch/tracker.yml create mode 100644 tasks/go-build.yml create mode 100644 tasks/go-test.yml create mode 100644 terraform/staging/main.tf diff --git a/pipelines/multi-branch/template.yml b/pipelines/multi-branch/template.yml new file mode 100644 index 0000000..b0bc54f --- /dev/null +++ b/pipelines/multi-branch/template.yml @@ -0,0 +1,81 @@ +resource_types: +- name: terraform + type: registry-image + source: + repository: ljfranklin/terraform-resource + +- name: gcs + type: registry-image + source: + repository: frodenas/gcs-resource + +resources: +- name: branch + type: git + source: + uri: https://github.com/concourse/examples + branch: ((branch)) + +- name: examples + type: git + source: + uri: https://github.com/concourse/examples + +- name: build-artifact + type: gcs + source: + bucket: concourse-examples + json_key: ((concourse_artifacts_json_key)) + regexp: multi-branch/features/((feature))/my-app-(.+)\.tgz + +- name: staging-env + type: terraform + source: + env_name: ((feature)) + backend_type: gcs + backend_config: + bucket: concourse-examples + prefix: multi-branch/terraform + credentials: ((concourse_artifacts_json_key)) + +jobs: +- name: test + plan: + - in_parallel: + - get: branch + trigger: true + - get: examples + - task: unit + file: examples/tasks/go-test.yml + input_mapping: {repo: branch} + params: {MODULE: apps/golang} + +- name: build + plan: + - in_parallel: + - get: branch + passed: [test] + trigger: true + - get: examples + - task: build + file: examples/tasks/go-build.yml + params: + MODULE: apps/golang + BINARY_NAME: my-app + input_mapping: {repo: branch} + - put: build-artifact + params: {file: "binary/my-app-*.tgz"} + +- name: deploy + plan: + - in_parallel: + - get: build-artifact + passed: [build] + trigger: true + - get: examples + - load_var: bundle_url + file: build-artifact/url + - put: staging-env + params: + terraform_source: examples/terraform/staging + vars: {bundle_url: ((.:bundle_url))} diff --git a/pipelines/multi-branch/tracker.yml b/pipelines/multi-branch/tracker.yml new file mode 100644 index 0000000..8850662 --- /dev/null +++ b/pipelines/multi-branch/tracker.yml @@ -0,0 +1,83 @@ +resource_types: +- name: git-branches + type: registry-image + source: + repository: aoldershaw/git-branches-resource + +resources: +- name: feature-branches + type: git-branches + source: + uri: https://github.com/concourse/examples + branch_regex: 'feature/(?P.*)' + +- name: examples + type: git + source: + uri: https://github.com/concourse/examples + +jobs: +- name: set-feature-pipelines + plan: + - in_parallel: + - get: feature-branches + trigger: true + - get: examples + - load_var: branches + file: feature-branches/branches.json + - across: + - var: branch + values: ((.:branches)) + set_pipeline: dev + file: examples/pipelines/multi-branch/template.yml + instance_vars: {feature: ((.:branch.groups.feature))} + vars: {branch: ((.:branch.name))} + +- name: cleanup-inactive-workspaces + plan: + - get: feature-branches + passed: [set-feature-pipelines] + trigger: true + - load_var: active_branches + file: feature-branches/branches.json + - task: cleanup + config: + platform: linux + image_resource: + type: registry-image + source: {repository: hashicorp/terraform} + params: + ACTIVE_BRANCHES: ((.:active_branches)) + TERRAFORM_BACKEND_CONFIG: + terraform: + backend: + gcs: + bucket: concourse-examples + prefix: multi-branch/terraform + credentials: ((concourse_artifacts_json_key)) + run: + path: sh + args: + - -c + - | + set -euo pipefail + + apk add jq + + active_features="$(echo "$ACTIVE_BRANCHES" | jq '[.[].groups.feature]')" + + echo "$TERRAFORM_BACKEND_CONFIG" > backend.tf.json + terraform init + active_workspaces="$(terraform workspace list | grep -v '^[*]' | tr -d ' ' | jq --raw-input --slurp 'split("\n") | map(select(. != ""))')" + + jq -nr "$active_workspaces - $active_features | .[]" | while read extra_workspace + do + echo "deleting workspace $extra_workspace" + terraform workspace select "$extra_workspace" + terraform init + + terraform destroy -auto-approve + + terraform workspace select default + terraform workspace delete "$extra_workspace" + done diff --git a/tasks/go-build.yml b/tasks/go-build.yml new file mode 100644 index 0000000..31c019a --- /dev/null +++ b/tasks/go-build.yml @@ -0,0 +1,28 @@ +platform: linux + +image_resource: + type: registry-image + source: { repository: golang } + +inputs: +- name: repo + path: . + +outputs: +- name: binary + +params: + MODULE: + BINARY_NAME: + +run: + path: sh + args: + - -ce + - | + root=$(pwd) + cd "$MODULE" + go build -o "/tmp/$BINARY_NAME" + + timestamp=$(date '+%Y%m%d%H%M%S') + tar czf "$root/binary/$BINARY_NAME-$timestamp.tgz" -C /tmp "$BINARY_NAME" diff --git a/tasks/go-test.yml b/tasks/go-test.yml new file mode 100644 index 0000000..2f8b24d --- /dev/null +++ b/tasks/go-test.yml @@ -0,0 +1,20 @@ +platform: linux + +image_resource: + type: registry-image + source: { repository: golang } + +inputs: +- name: repo + path: . + +params: + MODULE: + +run: + path: sh + args: + - -ce + - | + cd "$MODULE" + go test -v ./... diff --git a/terraform/staging/main.tf b/terraform/staging/main.tf new file mode 100644 index 0000000..b625d07 --- /dev/null +++ b/terraform/staging/main.tf @@ -0,0 +1,13 @@ +variable "bundle_url" { + type = string +} + +resource "null_resource" "fake_deployment" { + triggers = { + bundle_url = var.bundle_url + } + + provisioner "local-exec" { + command = "echo \"pretending to deploy from ${var.bundle_url}\"..." + } +} From 120b1a1cc0c802e676bfd0f6c18ada19f09a395f Mon Sep 17 00:00:00 2001 From: Aidan Oldershaw Date: Tue, 20 Jul 2021 11:37:31 -0400 Subject: [PATCH 2/4] use terraform resource to destroy old workspaces a bit torn on whether to do this, since it's possibly easier to follow just doing it in the shell script but hey, at least it uncovered a bug: https://github.com/concourse/concourse/pull/7310 Signed-off-by: Aidan Oldershaw --- pipelines/multi-branch/tracker.yml | 70 ++++++++++++++++++------------ terraform/staging/main.tf | 3 +- 2 files changed, 45 insertions(+), 28 deletions(-) diff --git a/pipelines/multi-branch/tracker.yml b/pipelines/multi-branch/tracker.yml index 8850662..82e0c19 100644 --- a/pipelines/multi-branch/tracker.yml +++ b/pipelines/multi-branch/tracker.yml @@ -4,6 +4,11 @@ resource_types: source: repository: aoldershaw/git-branches-resource +- name: terraform + type: registry-image + source: + repository: ljfranklin/terraform-resource + resources: - name: feature-branches type: git-branches @@ -16,6 +21,15 @@ resources: source: uri: https://github.com/concourse/examples +- name: staging-env + type: terraform + source: + backend_type: gcs + backend_config: &terraform_backend_config + bucket: concourse-examples + prefix: multi-branch/terraform + credentials: ((concourse_artifacts_json_key)) + jobs: - name: set-feature-pipelines plan: @@ -35,26 +49,24 @@ jobs: - name: cleanup-inactive-workspaces plan: - - get: feature-branches - passed: [set-feature-pipelines] - trigger: true - - load_var: active_branches - file: feature-branches/branches.json - - task: cleanup + - in_parallel: + - get: feature-branches + passed: [set-feature-pipelines] + trigger: true + - get: examples + - task: find-inactive-workspaces config: platform: linux image_resource: type: registry-image source: {repository: hashicorp/terraform} + inputs: + - name: feature-branches + outputs: + - name: extra-workspaces params: - ACTIVE_BRANCHES: ((.:active_branches)) TERRAFORM_BACKEND_CONFIG: - terraform: - backend: - gcs: - bucket: concourse-examples - prefix: multi-branch/terraform - credentials: ((concourse_artifacts_json_key)) + gcs: *terraform_backend_config run: path: sh args: @@ -62,22 +74,26 @@ jobs: - | set -euo pipefail - apk add jq + apk add -q jq - active_features="$(echo "$ACTIVE_BRANCHES" | jq '[.[].groups.feature]')" + active_features="$(jq '[.[].groups.feature]' feature-branches/branches.json)" - echo "$TERRAFORM_BACKEND_CONFIG" > backend.tf.json + jq -n "{terraform: {backend: $TERRAFORM_BACKEND_CONFIG}}" > backend.tf.json terraform init - active_workspaces="$(terraform workspace list | grep -v '^[*]' | tr -d ' ' | jq --raw-input --slurp 'split("\n") | map(select(. != ""))')" - jq -nr "$active_workspaces - $active_features | .[]" | while read extra_workspace - do - echo "deleting workspace $extra_workspace" - terraform workspace select "$extra_workspace" - terraform init - - terraform destroy -auto-approve + # List all active workspaces, ignoring the default workspace + active_workspaces="$(terraform workspace list | grep -v '^[*]' | tr -d ' ' | jq --raw-input --slurp 'split("\n") | map(select(. != ""))')" - terraform workspace select default - terraform workspace delete "$extra_workspace" - done + jq -n "$active_workspaces - $active_features" > extra-workspaces/workspaces.json + - load_var: extra_workspaces + file: extra-workspaces/workspaces.json + - across: + - var: workspace + values: ((.:extra_workspaces)) + put: staging-env + params: + terraform_source: examples/terraform/staging + env_name: ((.:workspace)) + action: destroy + get_params: + action: destroy diff --git a/terraform/staging/main.tf b/terraform/staging/main.tf index b625d07..df8fdaf 100644 --- a/terraform/staging/main.tf +++ b/terraform/staging/main.tf @@ -1,5 +1,6 @@ variable "bundle_url" { - type = string + type = string + default = "" } resource "null_resource" "fake_deployment" { From ec00928a4a73d95fd85917880fe39f1b33c50cde Mon Sep 17 00:00:00 2001 From: Aidan Oldershaw Date: Tue, 20 Jul 2021 16:51:37 -0400 Subject: [PATCH 3/4] rename service account key Signed-off-by: Aidan Oldershaw --- pipelines/multi-branch/template.yml | 4 ++-- pipelines/multi-branch/tracker.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pipelines/multi-branch/template.yml b/pipelines/multi-branch/template.yml index b0bc54f..3ffe2c3 100644 --- a/pipelines/multi-branch/template.yml +++ b/pipelines/multi-branch/template.yml @@ -25,7 +25,7 @@ resources: type: gcs source: bucket: concourse-examples - json_key: ((concourse_artifacts_json_key)) + json_key: ((gcp_service_account_key)) regexp: multi-branch/features/((feature))/my-app-(.+)\.tgz - name: staging-env @@ -36,7 +36,7 @@ resources: backend_config: bucket: concourse-examples prefix: multi-branch/terraform - credentials: ((concourse_artifacts_json_key)) + credentials: ((gcp_service_account_key)) jobs: - name: test diff --git a/pipelines/multi-branch/tracker.yml b/pipelines/multi-branch/tracker.yml index 82e0c19..9a644af 100644 --- a/pipelines/multi-branch/tracker.yml +++ b/pipelines/multi-branch/tracker.yml @@ -28,7 +28,7 @@ resources: backend_config: &terraform_backend_config bucket: concourse-examples prefix: multi-branch/terraform - credentials: ((concourse_artifacts_json_key)) + credentials: ((gcp_service_account_key)) jobs: - name: set-feature-pipelines From 34604c485f828c48830c84ad4001cfe039d369ef Mon Sep 17 00:00:00 2001 From: Aidan Oldershaw Date: Tue, 20 Jul 2021 16:55:07 -0400 Subject: [PATCH 4/4] explain named capture groups Signed-off-by: Aidan Oldershaw --- pipelines/multi-branch/tracker.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pipelines/multi-branch/tracker.yml b/pipelines/multi-branch/tracker.yml index 9a644af..73b2442 100644 --- a/pipelines/multi-branch/tracker.yml +++ b/pipelines/multi-branch/tracker.yml @@ -14,6 +14,11 @@ resources: type: git-branches source: uri: https://github.com/concourse/examples + # The "(?Ppattern)" syntax defines a named capture group. + # aoldershaw/git-branches-resource emits the value of each named capture + # group under the `groups` key. + # + # e.g. feature/some-feature ==> {"groups": {"feature": "some-feature"}} branch_regex: 'feature/(?P.*)' - name: examples