From 593f91ab36e3586e5410bf8a5254f632d29c3289 Mon Sep 17 00:00:00 2001 From: Taylor Silva Date: Fri, 2 Jul 2021 16:18:47 -0400 Subject: [PATCH 1/8] Add examples of templated pipelines Signed-off-by: Taylor Silva --- pipelines/templates/multiple-files/README.md | 7 ++ .../templates/multiple-files/jobs.lib.yml | 78 +++++++++++++++++++ .../multiple-files/resources.lib.yml | 25 ++++++ .../templates/multiple-files/template.yml | 19 +++++ pipelines/templates/multiple-files/vars.yml | 4 + pipelines/templates/simple/README.md | 1 + pipelines/templates/simple/template.yml | 13 ++++ pipelines/templates/simple/vars.yml | 6 ++ 8 files changed, 153 insertions(+) create mode 100644 pipelines/templates/multiple-files/README.md create mode 100644 pipelines/templates/multiple-files/jobs.lib.yml create mode 100644 pipelines/templates/multiple-files/resources.lib.yml create mode 100644 pipelines/templates/multiple-files/template.yml create mode 100644 pipelines/templates/multiple-files/vars.yml create mode 100644 pipelines/templates/simple/README.md create mode 100644 pipelines/templates/simple/template.yml create mode 100644 pipelines/templates/simple/vars.yml diff --git a/pipelines/templates/multiple-files/README.md b/pipelines/templates/multiple-files/README.md new file mode 100644 index 0000000..90034a9 --- /dev/null +++ b/pipelines/templates/multiple-files/README.md @@ -0,0 +1,7 @@ +This pipeline has been split over multiple files. This is an example so the only resource used is the [mock resource](https://github.com/concourse/mock-resource). + +Render the template using [ytt](https://github.com/vmware-tanzu/carvel-ytt/releases) and running the following command inside this directory: + +``` +ytt -f ./ +``` diff --git a/pipelines/templates/multiple-files/jobs.lib.yml b/pipelines/templates/multiple-files/jobs.lib.yml new file mode 100644 index 0000000..3bd5e15 --- /dev/null +++ b/pipelines/templates/multiple-files/jobs.lib.yml @@ -0,0 +1,78 @@ +#@ def unit_tests(): +name: unit-tests +plan: +- get: repo + trigger: true +- task: run-unit-tests + config: + platform: linux + image_resource: + type: mock + source: + mirror_self: true + inputs: + - name: repo + run: + path: sh + args: + - -c + - | + echo running the unit tests... + cat repo/branch.txt + sleep 4 + echo tests passed! +#@ end + +#@ def build_rc_image(): +name: build-image +plan: +- get: repo + trigger: true + passed: [unit-tests] +- task: build-image + config: + platform: linux + image_resource: + type: mock + source: + mirror_self: true + inputs: + - name: repo + outputs: + - name: image + run: + path: sh + args: + - -c + - | + echo building the image... + date +%Y-%m-%d > image/version + sleep 2 + echo image built! +- put: image-rc + params: + file: image/version +#@ end + +#@ def deploy(deployment_env): +name: #@ "deploy-" + deployment_env +plan: +- in_parallel: + - get: repo + passed: + #@ if deployment_env == "dev": + - build-image + #@ elif deployment_env == "prod": + - deploy-dev + #@ end + - get: image-rc + passed: + #@ if deployment_env == "dev": + - build-image + #@ elif deployment_env == "prod": + - deploy-dev + #@ end +- put: #@ deployment_env + "-env" + params: + file: image-rc/oci_image +#@ end diff --git a/pipelines/templates/multiple-files/resources.lib.yml b/pipelines/templates/multiple-files/resources.lib.yml new file mode 100644 index 0000000..f008d3d --- /dev/null +++ b/pipelines/templates/multiple-files/resources.lib.yml @@ -0,0 +1,25 @@ +#@ def resources(branch_name, artifact_slug): +- name: repo + icon: git + type: mock + source: + initial_version: #@ "repo-at-" + branch_name + create_files: + branch.txt: #@ branch_name + +- name: image-rc + icon: oci + type: mock + source: + initial_version: #@ artifact_slug + "-rc" + create_files: + oci_image: #@ artifact_slug + "-rc" + +- name: dev-env + icon: wrench + type: mock + +- name: prod-env + icon: cloud-check + type: mock +#@ end diff --git a/pipelines/templates/multiple-files/template.yml b/pipelines/templates/multiple-files/template.yml new file mode 100644 index 0000000..bf798c5 --- /dev/null +++ b/pipelines/templates/multiple-files/template.yml @@ -0,0 +1,19 @@ +#@ load("@ytt:data", "data") +#@ load("jobs.lib.yml", "unit_tests", "build_rc_image", "deploy") +#@ load("resources.lib.yml", "resources") + +jobs: +- #@ unit_tests() +- #@ build_rc_image() +- #@ deploy("dev") +- #@ deploy("prod") + +resources: #@ resources(data.values.branch_name, data.values.artifact_slug) + +#! TODO: temp fix to ensure usage of the latest mock resource +resource_types: +- name: mock + type: registry-image + source: + repository: concourse/mock-resource + diff --git a/pipelines/templates/multiple-files/vars.yml b/pipelines/templates/multiple-files/vars.yml new file mode 100644 index 0000000..6481a2f --- /dev/null +++ b/pipelines/templates/multiple-files/vars.yml @@ -0,0 +1,4 @@ +#@data/values +--- +branch_name: "feature-1024" +artifact_slug: "ft1024" diff --git a/pipelines/templates/simple/README.md b/pipelines/templates/simple/README.md new file mode 100644 index 0000000..1f2facc --- /dev/null +++ b/pipelines/templates/simple/README.md @@ -0,0 +1 @@ +To render the template download [ytt](https://github.com/vmware-tanzu/carvel-ytt/releases) and run `ytt -f pipeline.yml -f vars.yml > rendered.yml`. `rendered.yml` can then be passed to `fly set-pipeline` or a `set_pipeline` step to set the pipeline. diff --git a/pipelines/templates/simple/template.yml b/pipelines/templates/simple/template.yml new file mode 100644 index 0000000..509ef11 --- /dev/null +++ b/pipelines/templates/simple/template.yml @@ -0,0 +1,13 @@ +#@ load("@ytt:data", "data") +jobs: +- name: #@ data.values.job_name + "-job" + plan: + - task: #@ data.values.task_name + config: + platform: linux + image_resource: + type: mock + source: mirror_self: true + run: + path: echo + args: #@ data.values.args diff --git a/pipelines/templates/simple/vars.yml b/pipelines/templates/simple/vars.yml new file mode 100644 index 0000000..a3e44d0 --- /dev/null +++ b/pipelines/templates/simple/vars.yml @@ -0,0 +1,6 @@ +#@data/values +--- +job_name: "hello-world" +task_name: "hello-task" +args: + - "hello world of vars!" From c7e14c00406d3b3cb5abbecaf0b2e21f30a03921 Mon Sep 17 00:00:00 2001 From: Taylor Silva Date: Mon, 5 Jul 2021 12:24:22 -0400 Subject: [PATCH 2/8] Set rendered pipelines with ytt Signed-off-by: Taylor Silva --- pipelines/set-pipelines.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/pipelines/set-pipelines.yml b/pipelines/set-pipelines.yml index c01bdce..acb7970 100644 --- a/pipelines/set-pipelines.yml +++ b/pipelines/set-pipelines.yml @@ -3,6 +3,7 @@ resources: - name: concourse-examples type: git icon: github + check_every: 30m source: uri: https://github.com/concourse/examples @@ -15,6 +16,7 @@ jobs: trigger: true - set_pipeline: set-pipelines file: concourse-examples/pipelines/set-pipelines.yml + - name: set-example-pipelines public: true plan: @@ -67,3 +69,33 @@ jobs: file: concourse-examples/pipelines/php-larvel-app-testing.yml - set_pipeline: java file: concourse-examples/pipelines/java.yml + +- name: set-rendered-pipelines + public: true + plan: + - get: concourse-examples + trigger: true + passed: [set-self] + - task: render-pipelines + config: + platform: linux + image_resource: + type: registry-image + source: + repository: taylorsilva/carvel-ytt + inputs: + - name: concourse-examples + outputs: + - name: pipeline + run: + path: sh + args: + - -c + - | + ytt -f ./concourse-examples/pipelines/templates/simple > hello-world-rendered.yml + ytt -f ./concourse-examples/pipelines/templates/multiple-files > multi-files-rendered.yml + mv *.yml ./pipeline/ + - set_pipeline: hello-world-rendered + file: pipeline/hello-world-rendered.yml + - set_pipeline: multi-files-rendered + file: pipeline/multi-files-rendered.yml From 6753fd7195428aae589ff090e1c9acb4b63a5eb8 Mon Sep 17 00:00:00 2001 From: Taylor Silva Date: Mon, 5 Jul 2021 12:28:23 -0400 Subject: [PATCH 3/8] Fix bad yaml and error in the task correctly Signed-off-by: Taylor Silva --- pipelines/set-pipelines.yml | 2 +- pipelines/templates/simple/template.yml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pipelines/set-pipelines.yml b/pipelines/set-pipelines.yml index acb7970..e29cb41 100644 --- a/pipelines/set-pipelines.yml +++ b/pipelines/set-pipelines.yml @@ -90,7 +90,7 @@ jobs: run: path: sh args: - - -c + - -cx - | ytt -f ./concourse-examples/pipelines/templates/simple > hello-world-rendered.yml ytt -f ./concourse-examples/pipelines/templates/multiple-files > multi-files-rendered.yml diff --git a/pipelines/templates/simple/template.yml b/pipelines/templates/simple/template.yml index 509ef11..c81bffd 100644 --- a/pipelines/templates/simple/template.yml +++ b/pipelines/templates/simple/template.yml @@ -7,7 +7,8 @@ jobs: platform: linux image_resource: type: mock - source: mirror_self: true + source: + mirror_self: true run: path: echo args: #@ data.values.args From cf1c904d3269f57a93fbb784250c1f553bc47f01 Mon Sep 17 00:00:00 2001 From: Taylor Silva Date: Fri, 9 Jul 2021 17:35:50 -0400 Subject: [PATCH 4/8] Use "self" for main set-pipeline Signed-off-by: Taylor Silva --- pipelines/set-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/set-pipelines.yml b/pipelines/set-pipelines.yml index e29cb41..4a9ca3f 100644 --- a/pipelines/set-pipelines.yml +++ b/pipelines/set-pipelines.yml @@ -14,7 +14,7 @@ jobs: plan: - get: concourse-examples trigger: true - - set_pipeline: set-pipelines + - set_pipeline: self file: concourse-examples/pipelines/set-pipelines.yml - name: set-example-pipelines From 64a226a958cefab9dcf9193d82439f5bd7ca1e12 Mon Sep 17 00:00:00 2001 From: Aidan Oldershaw Date: Mon, 19 Jul 2021 16:48:32 -0400 Subject: [PATCH 5/8] add multi-branch workflow example Signed-off-by: Aidan Oldershaw --- pipelines/multi-branch/template.yml | 81 ++++++++++++++++++++++++++++ pipelines/multi-branch/tracker.yml | 83 +++++++++++++++++++++++++++++ tasks/go-build.yml | 28 ++++++++++ tasks/go-test.yml | 20 +++++++ terraform/staging/main.tf | 13 +++++ 5 files changed, 225 insertions(+) create mode 100644 pipelines/multi-branch/template.yml create mode 100644 pipelines/multi-branch/tracker.yml create mode 100644 tasks/go-build.yml create mode 100644 tasks/go-test.yml create mode 100644 terraform/staging/main.tf diff --git a/pipelines/multi-branch/template.yml b/pipelines/multi-branch/template.yml new file mode 100644 index 0000000..b0bc54f --- /dev/null +++ b/pipelines/multi-branch/template.yml @@ -0,0 +1,81 @@ +resource_types: +- name: terraform + type: registry-image + source: + repository: ljfranklin/terraform-resource + +- name: gcs + type: registry-image + source: + repository: frodenas/gcs-resource + +resources: +- name: branch + type: git + source: + uri: https://github.com/concourse/examples + branch: ((branch)) + +- name: examples + type: git + source: + uri: https://github.com/concourse/examples + +- name: build-artifact + type: gcs + source: + bucket: concourse-examples + json_key: ((concourse_artifacts_json_key)) + regexp: multi-branch/features/((feature))/my-app-(.+)\.tgz + +- name: staging-env + type: terraform + source: + env_name: ((feature)) + backend_type: gcs + backend_config: + bucket: concourse-examples + prefix: multi-branch/terraform + credentials: ((concourse_artifacts_json_key)) + +jobs: +- name: test + plan: + - in_parallel: + - get: branch + trigger: true + - get: examples + - task: unit + file: examples/tasks/go-test.yml + input_mapping: {repo: branch} + params: {MODULE: apps/golang} + +- name: build + plan: + - in_parallel: + - get: branch + passed: [test] + trigger: true + - get: examples + - task: build + file: examples/tasks/go-build.yml + params: + MODULE: apps/golang + BINARY_NAME: my-app + input_mapping: {repo: branch} + - put: build-artifact + params: {file: "binary/my-app-*.tgz"} + +- name: deploy + plan: + - in_parallel: + - get: build-artifact + passed: [build] + trigger: true + - get: examples + - load_var: bundle_url + file: build-artifact/url + - put: staging-env + params: + terraform_source: examples/terraform/staging + vars: {bundle_url: ((.:bundle_url))} diff --git a/pipelines/multi-branch/tracker.yml b/pipelines/multi-branch/tracker.yml new file mode 100644 index 0000000..8850662 --- /dev/null +++ b/pipelines/multi-branch/tracker.yml @@ -0,0 +1,83 @@ +resource_types: +- name: git-branches + type: registry-image + source: + repository: aoldershaw/git-branches-resource + +resources: +- name: feature-branches + type: git-branches + source: + uri: https://github.com/concourse/examples + branch_regex: 'feature/(?P.*)' + +- name: examples + type: git + source: + uri: https://github.com/concourse/examples + +jobs: +- name: set-feature-pipelines + plan: + - in_parallel: + - get: feature-branches + trigger: true + - get: examples + - load_var: branches + file: feature-branches/branches.json + - across: + - var: branch + values: ((.:branches)) + set_pipeline: dev + file: examples/pipelines/multi-branch/template.yml + instance_vars: {feature: ((.:branch.groups.feature))} + vars: {branch: ((.:branch.name))} + +- name: cleanup-inactive-workspaces + plan: + - get: feature-branches + passed: [set-feature-pipelines] + trigger: true + - load_var: active_branches + file: feature-branches/branches.json + - task: cleanup + config: + platform: linux + image_resource: + type: registry-image + source: {repository: hashicorp/terraform} + params: + ACTIVE_BRANCHES: ((.:active_branches)) + TERRAFORM_BACKEND_CONFIG: + terraform: + backend: + gcs: + bucket: concourse-examples + prefix: multi-branch/terraform + credentials: ((concourse_artifacts_json_key)) + run: + path: sh + args: + - -c + - | + set -euo pipefail + + apk add jq + + active_features="$(echo "$ACTIVE_BRANCHES" | jq '[.[].groups.feature]')" + + echo "$TERRAFORM_BACKEND_CONFIG" > backend.tf.json + terraform init + active_workspaces="$(terraform workspace list | grep -v '^[*]' | tr -d ' ' | jq --raw-input --slurp 'split("\n") | map(select(. != ""))')" + + jq -nr "$active_workspaces - $active_features | .[]" | while read extra_workspace + do + echo "deleting workspace $extra_workspace" + terraform workspace select "$extra_workspace" + terraform init + + terraform destroy -auto-approve + + terraform workspace select default + terraform workspace delete "$extra_workspace" + done diff --git a/tasks/go-build.yml b/tasks/go-build.yml new file mode 100644 index 0000000..31c019a --- /dev/null +++ b/tasks/go-build.yml @@ -0,0 +1,28 @@ +platform: linux + +image_resource: + type: registry-image + source: { repository: golang } + +inputs: +- name: repo + path: . + +outputs: +- name: binary + +params: + MODULE: + BINARY_NAME: + +run: + path: sh + args: + - -ce + - | + root=$(pwd) + cd "$MODULE" + go build -o "/tmp/$BINARY_NAME" + + timestamp=$(date '+%Y%m%d%H%M%S') + tar czf "$root/binary/$BINARY_NAME-$timestamp.tgz" -C /tmp "$BINARY_NAME" diff --git a/tasks/go-test.yml b/tasks/go-test.yml new file mode 100644 index 0000000..2f8b24d --- /dev/null +++ b/tasks/go-test.yml @@ -0,0 +1,20 @@ +platform: linux + +image_resource: + type: registry-image + source: { repository: golang } + +inputs: +- name: repo + path: . + +params: + MODULE: + +run: + path: sh + args: + - -ce + - | + cd "$MODULE" + go test -v ./... diff --git a/terraform/staging/main.tf b/terraform/staging/main.tf new file mode 100644 index 0000000..b625d07 --- /dev/null +++ b/terraform/staging/main.tf @@ -0,0 +1,13 @@ +variable "bundle_url" { + type = string +} + +resource "null_resource" "fake_deployment" { + triggers = { + bundle_url = var.bundle_url + } + + provisioner "local-exec" { + command = "echo \"pretending to deploy from ${var.bundle_url}\"..." + } +} From 120b1a1cc0c802e676bfd0f6c18ada19f09a395f Mon Sep 17 00:00:00 2001 From: Aidan Oldershaw Date: Tue, 20 Jul 2021 11:37:31 -0400 Subject: [PATCH 6/8] use terraform resource to destroy old workspaces a bit torn on whether to do this, since it's possibly easier to follow just doing it in the shell script but hey, at least it uncovered a bug: https://github.com/concourse/concourse/pull/7310 Signed-off-by: Aidan Oldershaw --- pipelines/multi-branch/tracker.yml | 70 ++++++++++++++++++------------ terraform/staging/main.tf | 3 +- 2 files changed, 45 insertions(+), 28 deletions(-) diff --git a/pipelines/multi-branch/tracker.yml b/pipelines/multi-branch/tracker.yml index 8850662..82e0c19 100644 --- a/pipelines/multi-branch/tracker.yml +++ b/pipelines/multi-branch/tracker.yml @@ -4,6 +4,11 @@ resource_types: source: repository: aoldershaw/git-branches-resource +- name: terraform + type: registry-image + source: + repository: ljfranklin/terraform-resource + resources: - name: feature-branches type: git-branches @@ -16,6 +21,15 @@ resources: source: uri: https://github.com/concourse/examples +- name: staging-env + type: terraform + source: + backend_type: gcs + backend_config: &terraform_backend_config + bucket: concourse-examples + prefix: multi-branch/terraform + credentials: ((concourse_artifacts_json_key)) + jobs: - name: set-feature-pipelines plan: @@ -35,26 +49,24 @@ jobs: - name: cleanup-inactive-workspaces plan: - - get: feature-branches - passed: [set-feature-pipelines] - trigger: true - - load_var: active_branches - file: feature-branches/branches.json - - task: cleanup + - in_parallel: + - get: feature-branches + passed: [set-feature-pipelines] + trigger: true + - get: examples + - task: find-inactive-workspaces config: platform: linux image_resource: type: registry-image source: {repository: hashicorp/terraform} + inputs: + - name: feature-branches + outputs: + - name: extra-workspaces params: - ACTIVE_BRANCHES: ((.:active_branches)) TERRAFORM_BACKEND_CONFIG: - terraform: - backend: - gcs: - bucket: concourse-examples - prefix: multi-branch/terraform - credentials: ((concourse_artifacts_json_key)) + gcs: *terraform_backend_config run: path: sh args: @@ -62,22 +74,26 @@ jobs: - | set -euo pipefail - apk add jq + apk add -q jq - active_features="$(echo "$ACTIVE_BRANCHES" | jq '[.[].groups.feature]')" + active_features="$(jq '[.[].groups.feature]' feature-branches/branches.json)" - echo "$TERRAFORM_BACKEND_CONFIG" > backend.tf.json + jq -n "{terraform: {backend: $TERRAFORM_BACKEND_CONFIG}}" > backend.tf.json terraform init - active_workspaces="$(terraform workspace list | grep -v '^[*]' | tr -d ' ' | jq --raw-input --slurp 'split("\n") | map(select(. != ""))')" - jq -nr "$active_workspaces - $active_features | .[]" | while read extra_workspace - do - echo "deleting workspace $extra_workspace" - terraform workspace select "$extra_workspace" - terraform init - - terraform destroy -auto-approve + # List all active workspaces, ignoring the default workspace + active_workspaces="$(terraform workspace list | grep -v '^[*]' | tr -d ' ' | jq --raw-input --slurp 'split("\n") | map(select(. != ""))')" - terraform workspace select default - terraform workspace delete "$extra_workspace" - done + jq -n "$active_workspaces - $active_features" > extra-workspaces/workspaces.json + - load_var: extra_workspaces + file: extra-workspaces/workspaces.json + - across: + - var: workspace + values: ((.:extra_workspaces)) + put: staging-env + params: + terraform_source: examples/terraform/staging + env_name: ((.:workspace)) + action: destroy + get_params: + action: destroy diff --git a/terraform/staging/main.tf b/terraform/staging/main.tf index b625d07..df8fdaf 100644 --- a/terraform/staging/main.tf +++ b/terraform/staging/main.tf @@ -1,5 +1,6 @@ variable "bundle_url" { - type = string + type = string + default = "" } resource "null_resource" "fake_deployment" { From ec00928a4a73d95fd85917880fe39f1b33c50cde Mon Sep 17 00:00:00 2001 From: Aidan Oldershaw Date: Tue, 20 Jul 2021 16:51:37 -0400 Subject: [PATCH 7/8] rename service account key Signed-off-by: Aidan Oldershaw --- pipelines/multi-branch/template.yml | 4 ++-- pipelines/multi-branch/tracker.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pipelines/multi-branch/template.yml b/pipelines/multi-branch/template.yml index b0bc54f..3ffe2c3 100644 --- a/pipelines/multi-branch/template.yml +++ b/pipelines/multi-branch/template.yml @@ -25,7 +25,7 @@ resources: type: gcs source: bucket: concourse-examples - json_key: ((concourse_artifacts_json_key)) + json_key: ((gcp_service_account_key)) regexp: multi-branch/features/((feature))/my-app-(.+)\.tgz - name: staging-env @@ -36,7 +36,7 @@ resources: backend_config: bucket: concourse-examples prefix: multi-branch/terraform - credentials: ((concourse_artifacts_json_key)) + credentials: ((gcp_service_account_key)) jobs: - name: test diff --git a/pipelines/multi-branch/tracker.yml b/pipelines/multi-branch/tracker.yml index 82e0c19..9a644af 100644 --- a/pipelines/multi-branch/tracker.yml +++ b/pipelines/multi-branch/tracker.yml @@ -28,7 +28,7 @@ resources: backend_config: &terraform_backend_config bucket: concourse-examples prefix: multi-branch/terraform - credentials: ((concourse_artifacts_json_key)) + credentials: ((gcp_service_account_key)) jobs: - name: set-feature-pipelines From 34604c485f828c48830c84ad4001cfe039d369ef Mon Sep 17 00:00:00 2001 From: Aidan Oldershaw Date: Tue, 20 Jul 2021 16:55:07 -0400 Subject: [PATCH 8/8] explain named capture groups Signed-off-by: Aidan Oldershaw --- pipelines/multi-branch/tracker.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pipelines/multi-branch/tracker.yml b/pipelines/multi-branch/tracker.yml index 9a644af..73b2442 100644 --- a/pipelines/multi-branch/tracker.yml +++ b/pipelines/multi-branch/tracker.yml @@ -14,6 +14,11 @@ resources: type: git-branches source: uri: https://github.com/concourse/examples + # The "(?Ppattern)" syntax defines a named capture group. + # aoldershaw/git-branches-resource emits the value of each named capture + # group under the `groups` key. + # + # e.g. feature/some-feature ==> {"groups": {"feature": "some-feature"}} branch_regex: 'feature/(?P.*)' - name: examples