dependency-builds.yml.erb

<%
require 'yaml'
require 'set'
config = YAML.load_file(File.join(__dir__, 'config', 'dependency-builds.yml'))

dependencies = config['dependencies']
cflinuxfs4_build_dependencies = config['cflinuxfs4_build_dependencies']
stacks = config['build_stacks']
skip_deprecation_check = config['skip_deprecation_check']

def common_prefix(stringA, stringB)
  max_len = 0
  (0..[stringA.size, stringB.size].min).each do |idx|
    if stringA[idx] == stringB[idx]
      max_len = idx+1
    else
      break
    end
  end
  stringA.slice(0,max_len)
end

def check_for_lines_overlap(lines_array)
  partition_hash = {}
  lines_array.each do|i|
    x_count = i.count('X')
    if partition_hash[x_count]
      partition_hash[x_count] += [i]
    else
      partition_hash[x_count] = [i]
    end
  end
  pairs = partition_hash.keys.combination(2).to_a.each do |p1, p2|
    partition_hash[p1].product(partition_hash[p2]).each do |strA, strB|
      return true if common_prefix(strA,strB) != ''
    end
  end
  false
end


def validate_dependencies(deps, skip_deprecation_check)
  deps.each do |dep_name, dep|
    if !(dep['buildpacks'].values.all? {|bp| bp['lines'].nil? } || dep['buildpacks'].values.all? {|bp| bp['lines'] })
      raise "For #{dep_name}, mixing buildpacks that specify version lines and ones that do not is NOT currently supported"
    end

  if check_for_lines_overlap(dep['buildpacks'].values.flat_map { |bp| get_version_lines(bp['lines'])})
      raise "For #{dep_name}, overlapping version lines are not supported (e.g. 1.X.X and 1.2.X)"
    end

  # validate dep dates -  if dep is not in skip_deprecation it must have deprecation dates for all version lines
    if is_multiline?(dep) && !(skip_deprecation_check.include?(dep_name))
      lines_missing = get_all_lines_missing_deprecation_dates(dep)
      if lines_missing.any?
        raise "For #{dep_name}, there are missing deprecation dates for these version lines: #{lines_missing}"
      end
    end

  end
end

def get_all_lines_missing_deprecation_dates(dep)
  # return an array of hashes which are lines missing deprecation dates across all relevant buildpacks for given dep
  dep['buildpacks'].values.reduce([]) {|sum, bp| sum | get_lines_missing_deprecation_dates(bp['lines'])}
end

def get_lines_missing_deprecation_dates(bp_lines)
  # return only the hashes missing deprecation dates from an array of hashes
  bp_lines&.select{|line_hash| line_hash['deprecation_date'].nil?}.map{|line_hash| line_hash['line']}
end

def get_version_lines(bp_lines, default_return = [])
  bp_lines&.map { |ln| ln['line']} || default_return
end

def is_multiline?(dep)
  dep['buildpacks'].values.map{ |ln| get_version_lines(ln['lines']).select {|ver| ver != 'latest'} }.flatten.any?
end

validate_dependencies(dependencies, skip_deprecation_check)

def get_version_line_type(lines)
  return nil if lines.nil?

  line = lines.first['line']
  x_idx = line.to_s.downcase.index('x')
  return nil unless x_idx
  case line.to_s[0, x_idx].count('.')
  when 1
    'major'
  when 2
    'minor'
  else
    nil
  end
end

def bp_uses_line?(bp_data, line)
  lines_present = get_version_lines(bp_data['lines']).nil?
  !lines_present && get_version_lines(bp_data['lines']).include?(line)
end

def get_git_uri(buildpack)
  bp_uri = "git@github.com:cloudfoundry/#{buildpack}-buildpack.git"
end

def get_git_repo(buildpack)
  bp_uri = "cloudfoundry/#{buildpack}-buildpack"
end
%>

resource_types:
- name: depwatcher
  type: docker-image
  source:
    repository: coredeps/depwatcher
- name: create-pull-request-resource
  type: docker-image
  source:
    repository: cfbuildpacks/create-pull-request-resource

resources:
- name: binary-builder
  type: git
  source:
    uri: {{binary-builder-git-uri}}
- name: buildpacks-ci
  type: git
  source:
    uri: https://github.com/cloudfoundry/buildpacks-ci
    branch: master
- name: builds
  type: git
  source:
    uri: git@github.com:cloudfoundry/public-buildpacks-ci-robots
    branch: main
    paths: [ binary-builds-new/ ]
    private_key: ((public-buildpacks-ci-robots-deploy-key.private_key))

<% stacks.each do |stack| %>
- name: <%= stack %>-image
  type: docker-image
  source: {repository: cloudfoundry/<%= stack %>}
<% end %>

- name: cflinuxfs3-dev-image
  type: docker-image
  source:
    repository: cfbuildpacks/cflinuxfs3-dev

<% dependencies.values.map{|dep| dep['buildpacks'].keys}.flatten.uniq.each do |buildpack|
branch = 'develop'
private_key = '((cf-buildpacks-eng-github-ssh-key.private_key))'
%>
- name: <%= buildpack %>-buildpack
  type: git
  source:
    uri: <%= get_git_uri(buildpack) %>
    private_key: <%= private_key %>
    branch: <%= branch %>
- name: <%= buildpack %>-buildpack-latest-released
  type: git
  source:
    uri: <%= get_git_uri(buildpack) %>
    private_key: <%= private_key %>
    branch: master
- name: <%= buildpack %>-pull-request
  type: create-pull-request-resource
  source:
    remote_repository: <%= get_git_repo(buildpack) %>
    github_token: ((buildpacks-github-token))
<% end %>

<% dependencies.each do |dep_name, dep| %>
<%
  lines = dep['buildpacks'].values.reduce([]) {|sum, bp| sum | get_version_lines(bp['lines'])}
  lines.push('latest') unless lines.include?('latest')
  lines.each do |line|
%>

- name: source-<%= dep_name.downcase %>-<%= line.downcase %>
  type: depwatcher
  check_every: 1h
  source:
    type: <%= dep['source_type'] || dep_name %>
    name: <%= dep_name %>
    <% if line != 'latest' %>
    version_filter: <%= line %>
    <% end %>
    <% (dep['source_params'] || []).each do |param| %>
    <%= param %>
    <% end %>
<% end %>

<% dep.fetch('monitored_deps', []).each do |monitored_dep_name| %>
- name: source-<%= monitored_dep_name.downcase %>-latest
  type: depwatcher
  check_every: 1h
  source:
    type: <%= monitored_dep_name %>
    name: <%= monitored_dep_name %>
<% end %>

<% if !dep['third_party_hosted'] %>
- name: buildpacks-bucket-<%= dep_name %>
  type: s3
  source:
    bucket: buildpacks.cloudfoundry.org
    access_key_id: ((buildpacks-cloudfoundry-org-aws-access-key-id))
    secret_access_key: ((buildpacks-cloudfoundry-org-aws-secret-access-key))
    # pip, go and libgdiplus have some versions of the form 'v1.2'; everyone else uses 'v1.2.3'
    regexp: dependencies/<%=dep_name%>/<%=dep_name == 'nginx-static' ? 'nginx' : dep_name %>.*?<%= (dep_name == 'pip' || dep_name == 'go' || dep_name == 'libgdiplus') ? '(\d+\.\d+(?:\.\d+)?)' : '(\d+\.\d+\.\d+)' %>(.*)
<% end %>
<% end %>
  ## Alerts ##

jobs:
<% dependencies.each do |dep_name, dep| %>
<%
skipped_version_lines_fs4 = dep['skip_lines_cflinuxfs4'] ? dep['skip_lines_cflinuxfs4'] : []
skipped_version_lines_fs3 = dep['skip_lines_cflinuxfs3'] ? dep['skip_lines_cflinuxfs3'] : []
version_lines = dep['buildpacks'].values.reduce([]) {|sum, bp| sum | get_version_lines(bp['lines'], ['latest'])}
%>
<% if is_multiline?(dep) %>
- name: create-<%= dep_name.downcase %>-new-line-story
  serial: true
  public: true
  plan:
  - in_parallel:
    - get: buildpacks-ci
    - get: source
      resource: source-<%= dep_name.downcase %>-latest
      trigger: true
      version: every
  - do:
    - task: create-new-version-line-story
      file: buildpacks-ci/tasks/create-new-version-line-story/create-new-version-line-story.yml
      params:
        BUILDPACKS: <%= dep['buildpacks'].keys.join(' ') %>
        EXISTING_VERSION_LINES: <%= version_lines.join(' ') %>
        GITHUB_TOKEN: ((buildpacks-github-token))
<% end %>

<% version_lines.each do |line| %>
<% if dep_name.downcase == 'node' && line.downcase == 'node-lts' %>
- name: create-node-node-lts-story
  serial: true
  public: true
  plan:
  - in_parallel:
    - get: buildpacks-ci
    - get: source
      resource: source-node-node-lts
      trigger: true
      version: every
    - get: builds
  - do:
    - task: create-new-version-line-story-node
      file: buildpacks-ci/tasks/create-new-version-line-story/create_node_lts.yml
      params:
        BUILDPACKS: <%= dep['buildpacks'].select{ |_, bp_data| bp_uses_line?(bp_data,line) }.keys.join(' ') %>
        GITHUB_TOKEN: ((buildpacks-github-token))
  <% else %>
  <% end %>


  <% dep['copy-stacks']&.each do |stack| %>
- name: copy-<%= dep_name.downcase %>-<%= line.downcase %>-<%= stack.downcase %>
  plan:
  - in_parallel:
    - get: buildpacks-ci
    - get: source
      resource: source-<%= dep_name.downcase %>-<%= line.downcase %>
      trigger: true
      version: every
      passed: [ build-<%= dep_name.downcase %>-<%= line.downcase %> ]
    - get: builds
  - task: repackage-dependency
    file: buildpacks-ci/tasks/repackage-dependency/task.yml
    params:
      STACK: <%= stack %>
      ANY_STACK: <%= dep['any_stack'] %>
  - put: bionic-builds-metadata
    resource: builds
    params:
      repository: builds
      rebase: true
  <% end %>

  <%
  specific_stack = cflinuxfs4_build_dependencies.include?(dep_name.downcase) ? stacks : [stacks.last]
  if dep_name.downcase == 'php'
    specific_stack = specific_stack - ['cflinuxfs3'] + ['cflinuxfs3-dev']
  elsif dep_name.downcase == 'bundler'
    specific_stack = ['cflinuxfs4']
  end
  %>
- name: build-<%= dep_name.downcase %>-<%= line.downcase %>
  serial: true
  public: true
  plan:
  - in_parallel:
    - get: buildpacks-ci
    - get: binary-builder
    - get: source
      resource: source-<%= dep_name.downcase %>-<%= line.downcase %>
      trigger: true
      version: every
    <% dep.fetch('monitored_deps', []).each do |monitored_dep_name| %>
    - get: source-<%= monitored_dep_name.downcase %>-latest
      trigger: true
      version: every
    <% end %>
  <% specific_stack.each do |stack| %>
    - get: <%= stack %>-image
  <% end %>
    - get: builds
  - in_parallel:
  <%
  build_stacks = dep['any_stack'] ? ['any-stack'] : specific_stack
  build_stacks = build_stacks - ['cflinuxfs4'] if skipped_version_lines_fs4.map(&:downcase).include?(line.downcase)
  build_stacks = build_stacks - ['cflinuxfs3'] if skipped_version_lines_fs3.map(&:downcase).include?(line.downcase)
  %>
  <% build_stacks.each do |stack| %>
    - do:
      - task: build-binary-<%= stack %>
        image: <%= stack == 'any-stack' ? (dep_name == 'bundler' ? 'cflinuxfs4' : 'cflinuxfs3') : stack %>-image
        file: buildpacks-ci/tasks/build-binary-new<%= (stack == 'cflinuxfs4' || (stack == 'any-stack' && dep_name == 'bundler')) ? '-cflinuxfs4' : '' %>/build.yml
        output_mapping: {artifacts: <%= stack %>-artifacts, builds-artifacts: <%= stack %>-builds-metadata}
        params:
          STACK: <%= stack %>
      - in_parallel:
    <% unless dep['third_party_hosted'] %>
        - put: buildpacks-bucket-<%= dep_name %>-<%= stack %>
          resource: buildpacks-bucket-<%= dep_name %>
          params:
            file: <%= stack %>-artifacts/<%= dep_name == "nginx-static" ? "nginx" : dep_name %>*
    <% end %>
        - put: builds-metadata-<%= stack %>
          resource: builds
          params:
            repository: <%= stack %>-builds-metadata
            rebase: true
  <% end %>
<% end %>

<% dep['buildpacks'].each do |bp_name, bp| %>
  <% bp.fetch('lines', []).each do |line_hash| %>
- name: update-<%= dep_name.downcase %>-<%= line_hash['line'].downcase %>-<%= bp_name.downcase %>
  serial: true
  public: true
  plan:
  - in_parallel:
    - get: buildpack
      resource: <%= bp_name %>-buildpack
    - get: buildpack-latest-released
      resource: <%= bp_name %>-buildpack-latest-released
    - get: buildpacks-ci
    - get: source
      resource: source-<%= dep_name.downcase %>-<%= line_hash['line'].downcase %>
      passed:
    <% if dep['copy-stacks'] %>
      <% dep['copy-stacks'].each do |copy_stack| %>
      - copy-<%= dep_name.downcase %>-<%= line_hash['line'].downcase %>-<%= copy_stack.downcase %>
      <% end %>
    <% else %>
      - build-<%= dep_name.downcase %>-<%= line_hash['line'].downcase %>
    <% end %>
      version: every
      trigger: true
    <% dep.fetch('monitored_deps', []).each do |monitored_dep_name| %>
    - get: source-<%= monitored_dep_name.downcase %>-latest
      trigger: true
      version: every
      passed: [<%= "build-#{dep_name.downcase}-#{line_hash['line'].downcase}" %>]
    <% end %>
    - get: builds
  - task: update-buildpack-dependency
    file: buildpacks-ci/tasks/update-buildpack-dependency/task.yml
    params:
      GPG_SIGNING_KEY_ID: ((cf-buildpacks-eng-gpg-key-id))
      GPG_SIGNING_KEY: ((cf-buildpacks-eng-gpg-key))
      VERSION_LINE: <%= line_hash['line'] %>
      VERSION_LINE_TYPE: <%= if dep_name == 'nginx' && bp_name == 'nginx' then 'nginx' else get_version_line_type(bp['lines']) end %>
      REMOVAL_STRATEGY: <%= bp.fetch('removal_strategy', 'remove_all') %>
      DEPRECATION_DATE: <%= line_hash['deprecation_date'] %>
      DEPRECATION_LINK: <%= line_hash['link'] %>
      DEPRECATION_MATCH: <%= line_hash['match'] %>
    output_mapping:
      artifacts: buildpack
  - put: <%= bp_name %>-pull-request
    params:
      repo_location: buildpack
      title: <%= "Updating version for #{dep_name} for #{line_hash['line']} " %>
      branch_prefix: "pr-by-releng-bot"
      auto_merge: false
      base: develop
  <% end %>
<% end %>
<% end %>

groups:
  - name: all
    jobs:
<% dependencies.each do |dep_name, dep| %>
  <% version_lines = dep['buildpacks'].values.reduce([]) {|sum, bp| sum | get_version_lines(bp['lines'], ['latest'])} %>
  <% if is_multiline?(dep) %>
    - create-<%= dep_name.downcase %>-new-line-story
  <% end %>
  <% version_lines.each do |line| %>
    <% if dep_name.downcase == 'node' && line.downcase == 'node-lts' %>
    - create-<%= dep_name.downcase %>-<%= line.downcase %>-story
    <% end %>
      <% dep['copy-stacks']&.each do |stack| %>
    - copy-<%= dep_name.downcase %>-<%= line.downcase %>-<%= stack.downcase %>
      <% end %>
    - build-<%= dep_name.downcase %>-<%= line.downcase %>
  <% end %>
    <% dep['buildpacks'].each do |bp_name, bp| %>
      <% get_version_lines(bp['lines'], ['latest']).each do |line| %>
    - update-<%= dep_name.downcase %>-<%= line.downcase %>-<%= bp_name.downcase %>
     <% end %>
  <% end %>
<% end %>

<% dependencies.each do |dep_name, dep| %>
  <% version_lines = dep['buildpacks'].values.reduce([]) {|sum, bp| sum | get_version_lines(bp['lines'], ['latest'])} %>
  - name: <%= dep_name.downcase %>
    jobs:
  <% if is_multiline?(dep) %>
    - create-<%= dep_name.downcase %>-new-line-story
  <% end %>
  <% version_lines.each do |line| %>
    <% if dep_name.downcase == 'node' && line.downcase == 'node-lts' %>
    - create-<%= dep_name.downcase %>-<%= line.downcase %>-story
    <% end %>
    <% dep['copy-stacks']&.each do |stack| %>
    - copy-<%= dep_name.downcase %>-<%= line.downcase %>-<%= stack.downcase %>
    <% end %>
    - build-<%= dep_name.downcase %>-<%= line.downcase %>
  <% end %>
  <% dep['buildpacks'].each do |bp_name, bp| %>
    <% get_version_lines(bp['lines'], ['latest']).each do |line| %>
    - update-<%= dep_name.downcase %>-<%= line.downcase %>-<%= bp_name.downcase %>
    <% end %>
  <% end %>
<% end %>