Merge branch 'concourse:master' into master

pipelines/multi-branch/template.yml

@@ -0,0 +1,81 @@
+resource_types:
+- name: terraform
+  type: registry-image
+  source:
+    repository: ljfranklin/terraform-resource
+
+- name: gcs
+  type: registry-image
+  source:
+    repository: frodenas/gcs-resource
+
+resources:
+- name: branch
+  type: git
+  source:
+    uri: https://github.com/concourse/examples
+    branch: ((branch))
+
+- name: examples
+  type: git
+  source:
+    uri: https://github.com/concourse/examples
+
+- name: build-artifact
+  type: gcs
+  source:
+    bucket: concourse-examples
+    json_key: ((gcp_service_account_key))
+    regexp: multi-branch/features/((feature))/my-app-(.+)\.tgz
+
+- name: staging-env
+  type: terraform
+  source:
+    env_name: ((feature))
+    backend_type: gcs
+    backend_config:
+      bucket: concourse-examples
+      prefix: multi-branch/terraform
+      credentials: ((gcp_service_account_key))
+
+jobs:
+- name: test
+  plan:
+  - in_parallel:
+    - get: branch
+      trigger: true
+    - get: examples
+  - task: unit
+    file: examples/tasks/go-test.yml
+    input_mapping: {repo: branch}
+    params: {MODULE: apps/golang}
+
+- name: build
+  plan:
+  - in_parallel:
+    - get: branch
+      passed: [test]
+      trigger: true
+    - get: examples
+  - task: build
+    file: examples/tasks/go-build.yml
+    params:
+      MODULE: apps/golang
+      BINARY_NAME: my-app
+    input_mapping: {repo: branch}
+  - put: build-artifact
+    params: {file: "binary/my-app-*.tgz"}
+
+- name: deploy
+  plan:
+  - in_parallel:
+    - get: build-artifact
+      passed: [build]
+      trigger: true
+    - get: examples
+  - load_var: bundle_url
+    file: build-artifact/url
+  - put: staging-env
+    params:
+      terraform_source: examples/terraform/staging
+      vars: {bundle_url: ((.:bundle_url))}

pipelines/multi-branch/tracker.yml

@@ -0,0 +1,104 @@
+resource_types:
+- name: git-branches
+  type: registry-image
+  source:
+    repository: aoldershaw/git-branches-resource
+
+- name: terraform
+  type: registry-image
+  source:
+    repository: ljfranklin/terraform-resource
+
+resources:
+- name: feature-branches
+  type: git-branches
+  source:
+    uri: https://github.com/concourse/examples
+    # The "(?P<name>pattern)" syntax defines a named capture group.
+    # aoldershaw/git-branches-resource emits the value of each named capture
+    # group under the `groups` key.
+    #
+    # e.g. feature/some-feature ==> {"groups": {"feature": "some-feature"}}
+    branch_regex: 'feature/(?P<feature>.*)'
+
+- name: examples
+  type: git
+  source:
+    uri: https://github.com/concourse/examples
+
+- name: staging-env
+  type: terraform
+  source:
+    backend_type: gcs
+    backend_config: &terraform_backend_config
+      bucket: concourse-examples
+      prefix: multi-branch/terraform
+      credentials: ((gcp_service_account_key))
+
+jobs:
+- name: set-feature-pipelines
+  plan:
+  - in_parallel:
+    - get: feature-branches
+      trigger: true
+    - get: examples
+  - load_var: branches
+    file: feature-branches/branches.json
+  - across:
+    - var: branch
+      values: ((.:branches))
+    set_pipeline: dev
+    file: examples/pipelines/multi-branch/template.yml
+    instance_vars: {feature: ((.:branch.groups.feature))}
+    vars: {branch: ((.:branch.name))}
+
+- name: cleanup-inactive-workspaces
+  plan:
+  - in_parallel:
+    - get: feature-branches
+      passed: [set-feature-pipelines]
+      trigger: true
+    - get: examples
+  - task: find-inactive-workspaces
+    config:
+      platform: linux
+      image_resource:
+        type: registry-image
+        source: {repository: hashicorp/terraform}
+      inputs:
+      - name: feature-branches
+      outputs:
+      - name: extra-workspaces
+      params:
+        TERRAFORM_BACKEND_CONFIG:
+          gcs: *terraform_backend_config
+      run:
+        path: sh
+        args:
+        - -c
+        - |
+          set -euo pipefail
+
+          apk add -q jq
+
+          active_features="$(jq '[.[].groups.feature]' feature-branches/branches.json)"
+
+          jq -n "{terraform: {backend: $TERRAFORM_BACKEND_CONFIG}}" > backend.tf.json
+          terraform init
+
+          # List all active workspaces, ignoring the default workspace
+          active_workspaces="$(terraform workspace list | grep -v '^[*]' | tr -d ' ' | jq --raw-input --slurp 'split("\n") | map(select(. != ""))')"
+
+          jq -n "$active_workspaces - $active_features" > extra-workspaces/workspaces.json
+  - load_var: extra_workspaces
+    file: extra-workspaces/workspaces.json
+  - across:
+    - var: workspace
+      values: ((.:extra_workspaces))
+    put: staging-env
+    params:
+      terraform_source: examples/terraform/staging
+      env_name: ((.:workspace))
+      action: destroy
+    get_params:
+      action: destroy

pipelines/set-pipelines.yml

@@ -3,6 +3,7 @@ resources:
 - name: concourse-examples
   type: git
   icon: github
+  check_every: 30m
   source:
     uri: https://github.com/concourse/examples
 
@@ -13,8 +14,9 @@ jobs:
   plan:
   - get: concourse-examples
     trigger: true
-  - set_pipeline: set-pipelines
+  - set_pipeline: self
     file: concourse-examples/pipelines/set-pipelines.yml
+
 - name: set-example-pipelines
   public: true
   plan:
@@ -67,3 +69,33 @@ jobs:
     file: concourse-examples/pipelines/php-larvel-app-testing.yml
   - set_pipeline: java
     file: concourse-examples/pipelines/java.yml
+
+- name: set-rendered-pipelines
+  public: true
+  plan:
+  - get: concourse-examples
+    trigger: true
+    passed: [set-self]
+  - task: render-pipelines
+    config:
+      platform: linux
+      image_resource:
+        type: registry-image
+        source:
+          repository: taylorsilva/carvel-ytt
+      inputs:
+      - name: concourse-examples
+      outputs:
+      - name: pipeline
+      run:
+        path: sh
+        args:
+        - -cx
+        - |
+          ytt -f ./concourse-examples/pipelines/templates/simple > hello-world-rendered.yml
+          ytt -f ./concourse-examples/pipelines/templates/multiple-files > multi-files-rendered.yml
+          mv *.yml ./pipeline/
+  - set_pipeline: hello-world-rendered
+    file: pipeline/hello-world-rendered.yml
+  - set_pipeline: multi-files-rendered
+    file: pipeline/multi-files-rendered.yml

pipelines/templates/multiple-files/README.md

@@ -0,0 +1,7 @@
+This pipeline has been split over multiple files. This is an example so the only resource used is the [mock resource](https://github.com/concourse/mock-resource).
+
+Render the template using [ytt](https://github.com/vmware-tanzu/carvel-ytt/releases) and running the following command inside this directory:
+
+```
+ytt -f ./
+```

pipelines/templates/multiple-files/jobs.lib.yml

@@ -0,0 +1,78 @@
+#@ def unit_tests():
+name: unit-tests
+plan:
+- get: repo
+  trigger: true
+- task: run-unit-tests
+  config:
+    platform: linux
+    image_resource:
+      type: mock
+      source:
+        mirror_self: true
+    inputs:
+    - name: repo
+    run:
+      path: sh
+      args:
+      - -c
+      - |
+        echo running the unit tests...
+        cat repo/branch.txt
+        sleep 4
+        echo tests passed!
+#@ end
+
+#@ def build_rc_image():
+name: build-image
+plan:
+- get: repo
+  trigger: true
+  passed: [unit-tests]
+- task: build-image
+  config:
+    platform: linux
+    image_resource:
+      type: mock
+      source:
+        mirror_self: true
+    inputs:
+    - name: repo
+    outputs:
+    - name: image
+    run:
+      path: sh
+      args:
+      - -c
+      - |
+        echo building the image...
+        date +%Y-%m-%d > image/version
+        sleep 2
+        echo image built!
+- put: image-rc
+  params:
+    file: image/version
+#@ end
+
+#@ def deploy(deployment_env):
+name: #@ "deploy-" + deployment_env
+plan:
+- in_parallel:
+  - get: repo
+    passed:
+    #@ if deployment_env == "dev":
+    - build-image
+    #@ elif deployment_env == "prod":
+    - deploy-dev
+    #@ end
+  - get: image-rc
+    passed:
+    #@ if deployment_env == "dev":
+    - build-image
+    #@ elif deployment_env == "prod":
+    - deploy-dev
+    #@ end
+- put: #@ deployment_env + "-env"
+  params:
+    file: image-rc/oci_image
+#@ end

pipelines/templates/multiple-files/resources.lib.yml

@@ -0,0 +1,25 @@
+#@ def resources(branch_name, artifact_slug):
+- name: repo
+  icon: git
+  type: mock
+  source:
+    initial_version: #@ "repo-at-" + branch_name
+    create_files:
+      branch.txt: #@ branch_name
+
+- name: image-rc
+  icon: oci
+  type: mock
+  source:
+    initial_version: #@ artifact_slug + "-rc"
+    create_files:
+      oci_image: #@ artifact_slug + "-rc"
+
+- name: dev-env
+  icon: wrench
+  type: mock
+
+- name: prod-env
+  icon: cloud-check
+  type: mock
+#@ end

pipelines/templates/multiple-files/template.yml

@@ -0,0 +1,19 @@
+#@ load("@ytt:data", "data")
+#@ load("jobs.lib.yml", "unit_tests", "build_rc_image", "deploy")
+#@ load("resources.lib.yml", "resources")
+
+jobs:
+- #@ unit_tests()
+- #@ build_rc_image()
+- #@ deploy("dev")
+- #@ deploy("prod")
+
+resources: #@ resources(data.values.branch_name, data.values.artifact_slug)
+
+#! TODO: temp fix to ensure usage of the latest mock resource
+resource_types:
+- name: mock
+  type: registry-image
+  source:
+    repository: concourse/mock-resource
+

pipelines/templates/multiple-files/vars.yml

@@ -0,0 +1,4 @@
+#@data/values
+---
+branch_name: "feature-1024"
+artifact_slug: "ft1024"

pipelines/templates/simple/README.md

@@ -0,0 +1 @@
+To render the template download [ytt](https://github.com/vmware-tanzu/carvel-ytt/releases) and run `ytt -f pipeline.yml -f vars.yml > rendered.yml`. `rendered.yml` can then be passed to `fly set-pipeline` or a `set_pipeline` step to set the pipeline.

pipelines/templates/simple/template.yml

@@ -0,0 +1,14 @@
+#@ load("@ytt:data", "data")
+jobs:
+- name: #@ data.values.job_name + "-job"
+  plan:
+  - task: #@ data.values.task_name
+    config:
+      platform: linux
+      image_resource:
+        type: mock
+        source:
+          mirror_self: true
+      run:
+        path: echo
+        args: #@ data.values.args

pipelines/templates/simple/vars.yml

@@ -0,0 +1,6 @@
+#@data/values
+---
+job_name: "hello-world"
+task_name: "hello-task"
+args:
+  - "hello world of vars!"

tasks/go-build.yml

@@ -0,0 +1,28 @@
+platform: linux
+
+image_resource:
+  type: registry-image
+  source: { repository: golang }
+
+inputs:
+- name: repo
+  path: .
+
+outputs:
+- name: binary
+
+params:
+  MODULE:
+  BINARY_NAME:
+
+run:
+  path: sh
+  args:
+  - -ce
+  - |
+    root=$(pwd)
+    cd "$MODULE"
+    go build -o "/tmp/$BINARY_NAME"
+
+    timestamp=$(date '+%Y%m%d%H%M%S')
+    tar czf "$root/binary/$BINARY_NAME-$timestamp.tgz" -C /tmp "$BINARY_NAME"

tasks/go-test.yml

@@ -0,0 +1,20 @@
+platform: linux
+
+image_resource:
+  type: registry-image
+  source: { repository: golang }
+
+inputs:
+- name: repo
+  path: .
+
+params:
+  MODULE:
+
+run:
+  path: sh
+  args:
+  - -ce
+  - |
+    cd "$MODULE"
+    go test -v ./...

terraform/staging/main.tf

@@ -0,0 +1,14 @@
+variable "bundle_url" {
+  type    = string
+  default = ""
+}
+
+resource "null_resource" "fake_deployment" {
+  triggers = {
+    bundle_url = var.bundle_url
+  }
+
+  provisioner "local-exec" {
+    command = "echo \"pretending to deploy from ${var.bundle_url}\"..."
+  }
+}