From 6cef53ed62edb88529c2d3a6204e58d0e419c273 Mon Sep 17 00:00:00 2001 From: Stephen Augustus Date: Sun, 27 Jan 2019 09:15:45 +0800 Subject: [PATCH] Add boilerplate for scripts (#80) * Add boilerplate templates and scripts Signed-off-by: Stephen Augustus * Remove unused ci/ folder Signed-off-by: Stephen Augustus * Update copyrights Signed-off-by: Stephen Augustus * Update deepcopy generator Signed-off-by: Stephen Augustus --- Dockerfile | 14 + Makefile | 19 +- ci/cleanup-e2e.sh | 9 - ci/dind-cluster-v1.12.sh | 2407 ----------------- ci/get-master-kubeconfig.sh | 17 - .../examples/azure/generate-yaml.sh | 14 + cmd/clusterctl/main.go | 16 + cmd/manager/main.go | 1 + hack/boilerplate/BUILD | 31 + hack/boilerplate/boilerplate.Dockerfile.txt | 14 + hack/boilerplate/boilerplate.Makefile.txt | 14 + hack/boilerplate/boilerplate.bzl.txt | 14 + hack/boilerplate/boilerplate.generatebzl.txt | 14 + hack/boilerplate/boilerplate.generatego.txt | 16 + hack/{ => boilerplate}/boilerplate.go.txt | 4 +- hack/boilerplate/boilerplate.py | 224 ++ hack/boilerplate/boilerplate.py.txt | 16 + hack/boilerplate/boilerplate.sh.txt | 14 + hack/boilerplate/boilerplate_test.py | 52 + hack/boilerplate/test/BUILD | 28 + hack/boilerplate/test/fail.go | 19 + hack/boilerplate/test/fail.py | 17 + hack/boilerplate/test/pass.go | 17 + hack/boilerplate/test/pass.py | 17 + hack/verify-boilerplate.sh | 35 + .../addtoscheme_azureprovider_v1alpha1.go | 1 + pkg/apis/apis.go | 3 +- pkg/apis/azureprovider/group.go | 2 +- .../azureclusterproviderconfig_types.go | 1 + .../azuremachineproviderconfig_types.go | 1 + pkg/apis/azureprovider/v1alpha1/doc.go | 1 + pkg/apis/azureprovider/v1alpha1/register.go | 1 + .../v1alpha1/zz_generated.deepcopy.go | 2 + pkg/cloud/azure/actuators/cluster/actuator.go | 3 + .../azure/actuators/cluster/actuator_test.go | 4 + pkg/cloud/azure/actuators/machine/actuator.go | 3 + .../azure/actuators/machine/actuator_test.go | 4 + pkg/cloud/azure/actuators/machine/status.go | 3 + pkg/cloud/azure/services/compute/disks.go | 3 + pkg/cloud/azure/services/compute/service.go | 3 + .../azure/services/compute/virtualmachines.go | 3 + pkg/cloud/azure/services/interfaces.go | 3 + pkg/cloud/azure/services/mock_interfaces.go | 3 + .../azure/services/mock_interfaces_helper.go | 3 + .../azure/services/network/interfaces.go | 3 + .../azure/services/network/publicipaddress.go | 3 + .../azure/services/network/securitygroups.go | 3 + pkg/cloud/azure/services/network/service.go | 3 + .../azure/services/network/virtualnetworks.go | 3 + .../azure/services/resources/deployments.go | 3 + pkg/cloud/azure/services/resources/groups.go | 3 + pkg/cloud/azure/services/resources/service.go | 3 + test/e2e/azure.go | 3 + test/e2e/e2e_test.go | 3 + test/e2e/kube.go | 3 + 55 files changed, 686 insertions(+), 2437 deletions(-) delete mode 100755 ci/cleanup-e2e.sh delete mode 100755 ci/dind-cluster-v1.12.sh delete mode 100755 ci/get-master-kubeconfig.sh create mode 100644 hack/boilerplate/BUILD create mode 100644 hack/boilerplate/boilerplate.Dockerfile.txt create mode 100644 hack/boilerplate/boilerplate.Makefile.txt create mode 100644 hack/boilerplate/boilerplate.bzl.txt create mode 100644 hack/boilerplate/boilerplate.generatebzl.txt create mode 100644 hack/boilerplate/boilerplate.generatego.txt rename hack/{ => boilerplate}/boilerplate.go.txt (92%) create mode 100755 hack/boilerplate/boilerplate.py create mode 100644 hack/boilerplate/boilerplate.py.txt create mode 100644 hack/boilerplate/boilerplate.sh.txt create mode 100644 hack/boilerplate/boilerplate_test.py create mode 100644 hack/boilerplate/test/BUILD create mode 100644 hack/boilerplate/test/fail.go create mode 100644 hack/boilerplate/test/fail.py create mode 100644 hack/boilerplate/test/pass.go create mode 100644 hack/boilerplate/test/pass.py create mode 100755 hack/verify-boilerplate.sh diff --git a/Dockerfile b/Dockerfile index ebf22075bdc..c23c0865485 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,17 @@ +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Build the manager binary FROM golang:1.11.5 as builder diff --git a/Makefile b/Makefile index b41b5668cfd..cb734f1e7bd 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,17 @@ +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Image URL to use all building/pushing image targets PREFIX ?= platform9 NAME = cluster-api-azure-provider-controller @@ -88,11 +102,14 @@ vet: lint: golangci-lint run +verify-boilerplate: + ./hack/verify-boilerplate.sh + # Generate code generate: go generate ./pkg/... ./cmd/... -check: bootstrap vendor-validate lint +check: verify-boilerplate bootstrap vendor-validate lint # Build the docker image docker-build: test diff --git a/ci/cleanup-e2e.sh b/ci/cleanup-e2e.sh deleted file mode 100755 index 55dc8066752..00000000000 --- a/ci/cleanup-e2e.sh +++ /dev/null @@ -1,9 +0,0 @@ -!/bin/bash -ROOT_DIR=$1 -# line in the cluster config file with the resource group name -RG_LINE=16 -rgName=$(cut -d ":" -f 2 <<<$(head -$RG_LINE $ROOT_DIR/cmd/clusterctl/examples/azure/out/cluster.yaml | tail -1) | tr -d '"') -exists=$(az group exists -n $rgName) -if [ "$exists" = true ]; then - az group delete -n $rgName --yes -fi diff --git a/ci/dind-cluster-v1.12.sh b/ci/dind-cluster-v1.12.sh deleted file mode 100755 index 29d5d7a215f..00000000000 --- a/ci/dind-cluster-v1.12.sh +++ /dev/null @@ -1,2407 +0,0 @@ -#!/bin/bash -# Copyright 2017 Mirantis -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -o errexit -set -o nounset -set -o pipefail -set -o errtrace - -if [ $(uname) = Darwin ]; then - readlinkf(){ perl -MCwd -e 'print Cwd::abs_path shift' "$1";} -else - readlinkf(){ readlink -f "$1"; } -fi -DIND_ROOT="$(cd $(dirname "$(readlinkf "${BASH_SOURCE}")"); pwd)" - -RUN_ON_BTRFS_ANYWAY="${RUN_ON_BTRFS_ANYWAY:-}" -if [[ ! ${RUN_ON_BTRFS_ANYWAY} ]] && docker info| grep -q '^Storage Driver: btrfs'; then - echo "ERROR: Docker is using btrfs storage driver which is unsupported by kubeadm-dind-cluster" >&2 - echo "Please refer to the documentation for more info." >&2 - echo "Set RUN_ON_BTRFS_ANYWAY to non-empty string to continue anyway." >&2 - exit 1 -fi - -# In case of linuxkit / moby linux, -v will not work so we can't -# mount /lib/modules and /boot. Also we'll be using localhost -# to access the apiserver. -using_linuxkit= -if ! docker info|grep -s '^Operating System: .*Docker for Windows' > /dev/null 2>&1 ; then - if docker info|grep -s '^Kernel Version: .*-moby$' >/dev/null 2>&1 || - docker info|grep -s '^Kernel Version: .*-linuxkit' > /dev/null 2>&1 ; then - using_linuxkit=1 - fi -fi - -# Determine when using Linux and docker daemon running locally -using_local_linuxdocker= -if [[ $(uname) == Linux && -z ${DOCKER_HOST:-} ]]; then - using_local_linuxdocker=1 -fi - -EMBEDDED_CONFIG=y;DIND_IMAGE=mirantis/kubeadm-dind-cluster:v1.12 - -# dind::localhost provides the local host IP based on the address family used for service subnet. -function dind::localhost() { - if [[ ${SERVICE_NET_MODE} = "ipv6" ]]; then - echo '[::1]' - else - echo '127.0.0.1' - fi -} - -# dind::family-for indicates whether the CIDR or IP is for an IPv6 or IPv4 family. -function dind::family-for { - local addr=$1 - if [[ "$addr" = *":"* ]]; then - echo "ipv6" - else - echo "ipv4" - fi -} - -# dind::cluster-suffix builds a suffix used for resources, based on the DIND_LABEL. -function dind::cluster-suffix { - if [ "$DIND_LABEL" != "$DEFAULT_DIND_LABEL" ]; then - echo "-${DIND_LABEL}" - else - echo '' - fi -} - -function dind::net-name { - echo "kubeadm-dind-net$( dind::cluster-suffix )" -} - -# dind::add-cluster will inject the cluster ID to the IP address. For IPv4, it is -# assumed that the IP is a /24 with the third part of the address available for cluster ID. -# For IPv6, it is assumed that there is enough space for the cluster to be added, and the -# cluster ID will be added to the 16 bits before the double colon. For example: -# -# 10.192.0.0/24 => 10.192.5.0/24 -# fd00:77:20::/64 => fd00:77:20:5::/64 -# -# This function is intended to be used for management networks. -# -# TODO: Validate that there is enough space for cluster ID. -# TODO: For IPv6 could get fancy and handle case where cluster ID is placed in upper 8 bits of hextet -# TODO: Consider if want to do /16 for IPv4 management subnet. -# -function dind::add-cluster { - local cidr=$1 - local ip_mode=$2 - - if [[ ${ip_mode} = "ipv4" ]]; then - echo ${cidr} | sed "s/^\([0-9]*\.[0-9]*\.\).*\/24$/\1${CLUSTER_ID}.0\/24/" - else # IPv6 - echo ${cidr} | sed "s/^\(.*\)\(\:\:\/[0-9]*\)$/\1:${CLUSTER_ID}\2/" - fi -} - -# dind::get-and-validate-cidrs takes a list of CIDRs and validates them based on the ip -# mode, returning them. For IPv4 only and IPv6 only modes, only one CIDR is expected. For -# dual stack, two CIDRS are expected. It verifies that the CIDRs are the right family and -# will use the provided defaults, when CIDRs are missing. For dual-stack, the IPv4 address -# will be first. -# -# For the management network, the cluster ID will be injected into the CIDR. Also, if no -# MGMT_CIDRS value is specified, but the legacy DIND_SUBNET/DIND_SUBNET_SIZE is provided, -# that will be used for the (first) CIDR. -# -# NOTE: It is expected that the CIDR size is /24 for IPv4 management networks. -# -# For pod CIDRs, the size will be increased by 8, to leave room for the node ID to be -# injected into the address. -# -# NOTE: For IPv4, the pod size is expected to be /16 -> /24 in usage. -# -function dind::get-and-validate-cidrs { - IFS=', ' read -r -a cidrs <<< "$1" - IFS=', ' read -r -a defaults <<< "$2" - local is_mgmt=$3 - case ${IP_MODE} in - ipv4) - case ${#cidrs[@]} in - 0) - cidrs[0]="${defaults[0]}" - ;; - 1) - ;; - *) - echo "ERROR! More than one CIDR provided '$1'" - exit 1 - ;; - esac - if [[ $( dind::family-for "${cidrs[0]}" ) != "ipv4" ]]; then - echo "ERROR! CIDR must be IPv4 value" - exit 1 - fi - if [[ ${is_mgmt} = true ]]; then - cidrs[0]="$( dind::add-cluster "${cidrs[0]}" "${IP_MODE}" )" - fi - ;; - - ipv6) - case ${#cidrs[@]} in - 0) - cidrs[0]="${defaults[0]}" - ;; - 1) - ;; - *) - echo "ERROR! More than one CIDR provided '$1'" - exit 1 - ;; - esac - if [[ $( dind::family-for "${cidrs[0]}" ) != "ipv6" ]]; then - echo "ERROR! CIDR must be IPv6 value" - exit 1 - fi - if [[ ${is_mgmt} = true ]]; then - cidrs[0]="$( dind::add-cluster "${cidrs[0]}" "${IP_MODE}" )" - fi - ;; - - dual-stack) - case ${#cidrs[@]} in - 0) - cidrs[0]="${defaults[0]}" - cidrs[1]="${defaults[1]}" - ;; - 1) - if [[ $( dind::family-for "${cidrs[0]}" ) = "ipv6" ]]; then - cidrs[1]=${cidrs[0]} - cidrs[0]="${defaults[0]}" # Assuming first default is a V4 address - else - cidrs[1]="${defaults[1]}" - fi - ;; - 2) - # Force ordering to have V4 address first - if [[ $( dind::family-for "${cidrs[0]}" ) = "ipv6" ]]; then - local temp=${cidrs[0]} - cidrs[0]=${cidrs[1]} - cidrs[1]=${temp} - fi - ;; - *) - echo "ERROR! More than two CIDRs provided '$1'" - exit 1 - ;; - esac - local have_v4="" - local have_v6="" - for cidr in ${cidrs[@]}; do - if [[ $( dind::family-for "${cidr}" ) = "ipv6" ]]; then - have_v6=1 - else - have_v4=1 - fi - done - if [[ -z ${have_v4} ]]; then - echo "ERROR! Missing IPv4 CIDR in '$1'" - exit 1 - fi - if [[ -z ${have_v6} ]]; then - echo "ERROR! Missing IPv6 CIDR in '$1'" - exit 1 - fi - if [[ ${is_mgmt} = true ]]; then - cidrs[0]="$( dind::add-cluster "${cidrs[0]}" "${IP_MODE}" )" - cidrs[1]="$( dind::add-cluster "${cidrs[1]}" "${IP_MODE}" )" - fi - ;; - esac - echo "${cidrs[@]}" -} - -# dind::make-ip-from-cidr strips off the slash and size, and appends the -# interface part to the prefix to form an IP. For IPv4, it strips off the -# fourth part of the prefix, so that it can be replaced. It assumes that the -# resulting prefix will be of sufficient size. It also will use hex for the -# appended part for IPv6, and decimal for IPv4. -# -# fd00:20::/64 -> fd00:20::a -# 10.96.0.0/12 -> 10.96.0.10 -# -function dind::make-ip-from-cidr { - prefix="$(echo $1 | sed 's,/.*,,')" - if [[ $( dind::family-for ${prefix} ) == "ipv4" ]]; then - printf "%s%d" $( echo ${prefix} | sed 's/0$//' ) $2 - else - printf "%s%x" ${prefix} $2 - fi -} - -# dind::add-cluster-id-and-validate-nat64-prefix will modify the IPv4 mapping -# subnet prefix, by adding the cluster ID (default 0) to the second octet. -# It will produce an error, if the prefix is not in the 10.0.0.0/8 or -# 172.16.0.0/12 private networks. -function dind::add-cluster-id-and-validate-nat64-prefix { - local parts - IFS="." read -a parts <<<${NAT64_V4_SUBNET_PREFIX} - if [[ ${#parts[@]} -ne 2 ]]; then - echo "ERROR! NAT64_V4_SUBNET_PREFIX must be two octets (have '${NAT64_V4_SUBNET_PREFIX}')" - exit 1 - fi - (( parts[1]+=${CLUSTER_ID} )) - NAT64_V4_SUBNET_PREFIX="${parts[0]}.${parts[1]}" - echo "Added cluster ID offset (${CLUSTER_ID}) to NAT64_V4_SUBNET_PREFIX giving prefix '${NAT64_V4_SUBNET_PREFIX}'" - if [[ ${parts[0]} -eq 10 ]]; then - if [[ ${parts[1]} > 253 ]]; then - echo "ERROR! NAT64_V4_SUBNET_PREFIX is too large for 10.0.0.0/8 private net" - exit 1 - fi - elif [[ ${parts[0]} -eq 172 ]]; then - if [[ ${parts[1]} -lt 16 || ${parts[1]} -gt 31 ]]; then - echo "ERROR! NAT64_V4_SUBNET_PREFIX is outside of range for 172.16.0.0/12 private net" - exit 1 - fi - else - echo "ERROR! NAT64_V4_SUBNET_PREFIX is not in 10.0.0.0/8 or 172.16.0.0/12 private networks" - exit 1 - fi - echo "Using NAT64 V4 mapping network prefix: ${NAT64_V4_SUBNET_PREFIX}" -} - - -# START OF PROCESSING... - -IP_MODE="${IP_MODE:-ipv4}" # ipv4, ipv6, dual-stack -# FUTURE: Once dual-stack support is released, check K8s version, and reject for older versions. -if [[ ! ${EMBEDDED_CONFIG:-} ]]; then - source "${DIND_ROOT}/config.sh" -fi - -# Multicluster support -# Users can specify a cluster ID number from 1..254, represented as a string. -# This will be used to form resource names "cluster-#", and will be used in the -# management subnet to give unique networks for each cluster. If the cluster ID -# is not specified, or zero, it will be considered a single cluster or the first -# in the multi-cluster. This is the recommended usage. -# -# For legacy support, the user can specify DIND_LABEL, which will be used in the -# resource names. If a cluster ID is specified (a hybrid case, where people are -# using the new method, but want custom names), the resourse name will have the -# suffix "-#" with the cluster ID. If no cluster ID is specified (for backward -# compatibility), then the resource name will be just the DIND_LABEL, and a pseudo- -# random number from 1..13 will be generated for the cluster ID to be used in -# management network. The range is limited, because, in IPv6 mode, the cluster ID -# is used in the NAT64 V4 subnet prefix, which must be in a private network. -# The default is 172.18, so the cluster ID cannot be larger than 13 to guarantee -# a valid value. -# -# To get around that limitation, you can set the cluster ID, in addition to the -# DIND_LABEL, and optionally, change the NAT64_V4_SUBNET_PREFIX value. -# -DEFAULT_DIND_LABEL='mirantis.kubeadm_dind_cluster_runtime' -if [[ -z ${DIND_LABEL+x} ]]; then # No legacy DIND_LABEL set - if [[ -z ${CLUSTER_ID+x} ]]; then # No cluster ID set - DIND_LABEL=${DEFAULT_DIND_LABEL} # Single cluster mode - CLUSTER_ID="0" - else # Have cluster ID - if [[ ${CLUSTER_ID} = "0" ]]; then - DIND_LABEL=${DEFAULT_DIND_LABEL} # Single cluster mode or first cluster of multi-cluster - else - DIND_LABEL="cluster-${CLUSTER_ID}" # Multi-cluster - fi - fi -else # Legacy DIND_LABEL set for multi-cluster - if [[ -z ${CLUSTER_ID+x} ]]; then # No cluster ID set, make one from 1..13, but don't use in resource names - CLUSTER_ID="$(( ($RANDOM % 12) + 1 ))" - else - if [[ ${CLUSTER_ID} = "0" ]]; then - CLUSTER_ID="$(( ($RANDOM % 12) + 1 ))" # Force a pseudo-random cluster for additional legacy cluster - else - DIND_LABEL="${DIND_LABEL}-${CLUSTER_ID}" - fi - fi -fi - -CNI_PLUGIN="${CNI_PLUGIN:-bridge}" -GCE_HOSTED="${GCE_HOSTED:-}" -DIND_ALLOW_AAAA_USE="${DIND_ALLOW_AAAA_USE:-}" # Default is to use DNS64 always for IPv6 mode -KUBE_ROUTER_VERSION="${KUBE_ROUTER_VERSION:-v0.2.0}" - -# Use legacy DIND_SUBNET/DIND_SUBNET_SIZE, only if MGMT_CIDRS is not set. -legacy_mgmt_cidr="" -if [[ ${DIND_SUBNET:-} && ${DIND_SUBNET_SIZE:-} ]]; then - legacy_mgmt_cidr="${DIND_SUBNET}/${DIND_SUBNET_SIZE}" -fi - -if [[ ${IP_MODE} = "dual-stack" ]]; then - mgmt_net_defaults="10.192.0.0/24, fd00:20::/64" - - KUBE_RSYNC_ADDR="${KUBE_RSYNC_ADDR:-::1}" - SERVICE_CIDR="${SERVICE_CIDR:-fd00:30::/110}" # Will default to IPv6 service net family - - pod_net_defaults="10.244.0.0/16, fd00:40::/72" - - USE_HAIRPIN="${USE_HAIRPIN:-true}" # Default is to use hairpin for dual-stack - DIND_ALLOW_AAAA_USE=true # Forced, so can access external hosts via IPv6 - if [[ ${DIND_ALLOW_AAAA_USE} && ${GCE_HOSTED} ]]; then - echo "ERROR! GCE does not support use of IPv6 for external addresses - aborting." - exit 1 - fi -elif [[ ${IP_MODE} = "ipv6" ]]; then - mgmt_net_defaults="fd00:20::/64" - - KUBE_RSYNC_ADDR="${KUBE_RSYNC_ADDR:-::1}" - SERVICE_CIDR="${SERVICE_CIDR:-fd00:30::/110}" - - pod_net_defaults="fd00:40::/72" - - USE_HAIRPIN="${USE_HAIRPIN:-true}" # Default is to use hairpin for IPv6 - if [[ ${DIND_ALLOW_AAAA_USE} && ${GCE_HOSTED} ]]; then - echo "ERROR! GCE does not support use of IPv6 for external addresses - aborting." - exit 1 - fi -else # IPv4 mode - mgmt_net_defaults="10.192.0.0/24" - - KUBE_RSYNC_ADDR="${KUBE_RSYNC_ADDR:-127.0.0.1}" - SERVICE_CIDR="${SERVICE_CIDR:-10.96.0.0/12}" - - pod_net_defaults="10.244.0.0/16" - - USE_HAIRPIN="${USE_HAIRPIN:-false}" # Disabled for IPv4, as issue with Virtlet networking - if [[ ${DIND_ALLOW_AAAA_USE} ]]; then - echo "WARNING! The DIND_ALLOW_AAAA_USE option is for IPv6 mode - ignoring setting." - DIND_ALLOW_AAAA_USE= - fi - if [[ ${CNI_PLUGIN} = "calico" || ${CNI_PLUGIN} = "calico-kdd" ]]; then - pod_net_defaults="192.168.0.0/16" - fi -fi - -IFS=' ' read -r -a mgmt_net_cidrs <<<$( dind::get-and-validate-cidrs "${MGMT_CIDRS:-${legacy_mgmt_cidr}}" "${mgmt_net_defaults[@]}" true ) - -REMOTE_DNS64_V4SERVER="${REMOTE_DNS64_V4SERVER:-8.8.8.8}" -if [[ ${IP_MODE} == "ipv6" ]]; then - # Uses local DNS64 container - dns_server="$( dind::make-ip-from-cidr ${mgmt_net_cidrs[0]} 0x100 )" - DNS64_PREFIX="${DNS64_PREFIX:-fd00:10:64:ff9b::}" - DNS64_PREFIX_SIZE="${DNS64_PREFIX_SIZE:-96}" - DNS64_PREFIX_CIDR="${DNS64_PREFIX}/${DNS64_PREFIX_SIZE}" - - LOCAL_NAT64_SERVER="$( dind::make-ip-from-cidr ${mgmt_net_cidrs[0]} 0x200 )" - NAT64_V4_SUBNET_PREFIX="${NAT64_V4_SUBNET_PREFIX:-172.18}" - dind::add-cluster-id-and-validate-nat64-prefix -else - dns_server="${REMOTE_DNS64_V4SERVER}" -fi - -SERVICE_NET_MODE="$( dind::family-for ${SERVICE_CIDR} )" -DNS_SVC_IP="$( dind::make-ip-from-cidr ${SERVICE_CIDR} 10 )" - -ETCD_HOST="${ETCD_HOST:-$( dind::localhost )}" - -IFS=' ' read -r -a pod_net_cidrs <<<$( dind::get-and-validate-cidrs "${POD_NETWORK_CIDR:-}" "${pod_net_defaults[@]}" false ) - -declare -a pod_prefixes -declare -a pod_sizes -# Extract the prefix and size from the provided pod CIDR(s), based on the IP mode of each. The -# size will be increased by 8, to make room for the node ID to be added to the prefix later. -# Bridge and PTP plugins can process IPv4 and IPv6 pod CIDRs, other plugins must be IPv4 only. -for pod_cidr in "${pod_net_cidrs[@]}"; do - if [[ $( dind::family-for "${pod_cidr}" ) = "ipv4" ]]; then - actual_size=$( echo ${pod_cidr} | sed 's,.*/,,' ) - if [[ ${actual_size} -ne 16 ]]; then - echo "ERROR! For IPv4 CIDRs, the size must be /16. Have '${pod_cidr}'" - exit 1 - fi - pod_sizes+=( 24 ) - pod_prefixes+=( "$(echo ${pod_cidr} | sed 's/^\([0-9]*\.[0-9]*\.\).*/\1/')" ) - else # IPv6 - if [[ ${CNI_PLUGIN} != "bridge" && ${CNI_PLUGIN} != "ptp" ]]; then - echo "ERROR! IPv6 pod networks are only supported by bridge and PTP CNI plugins" - exit 1 - fi - # There are several cases to address. First, is normal split of prefix and size: - # fd00:10:20:30::/64 ---> fd00:10:20:30: /72 - # - # Second, is when the prefix needs to be padded, so that node ID can be added later: - # fd00:10::/64 ---> fd00:10:0:0: /72 - # - # Third, is when the low order part of the address, must be removed for the prefix, - # as the node ID will be placed in the lower byte: - # fd00:10:20:30:4000::/72 ---> fd00:10:20:30:40 /80 - # - # We will attempt to check for three error cases. One is when the address part is - # way too big for the size specified: - # fd00:10:20:30:40::/48 ---> fd00:10:20: /56 desired, but conflict with 30:40: - # - # Another is when the address part, once trimmed for the size, would loose info: - # fd00:10:20:1234::/56 ---> fd00:10:20:12 /64, but lost 34:, which conflicts - # - # Lastly, again, trimming would leave high byte in hextet, conflicting with - # the node ID: - # fd00:10:20:30:1200::/64 ---> fd00:10:20:30:12 /72, but 12 conflicts - # - # Note: later, the node ID will be appended to the prefix generated. - # - cluster_size="$(echo ${pod_cidr} | sed 's,.*::/,,')" - pod_sizes+=( $((${cluster_size}+8)) ) - - pod_prefix="$(echo ${pod_cidr} | sed 's,::/.*,:,')" - num_colons="$(grep -o ":" <<< "${pod_prefix}" | wc -l)" - need_zero_pads=$((${cluster_size}/16)) - - if [[ ${num_colons} -gt $((need_zero_pads + 1)) ]]; then - echo "ERROR! Address part of CIDR (${pod_prefix}) is too large for /${cluster_size}" - exit 1 - fi - if [[ ${num_colons} -gt ${need_zero_pads} ]]; then - # Will be replacing lowest byte with node ID, so pull off lower byte and colon - if [[ ${pod_prefix: -3} != "00:" ]]; then # last byte is not zero - echo "ERROR! Cannot trim address part of CIDR (${pod_prefix}) to fit in /${cluster_size}" - exit 1 - fi - pod_prefix=${pod_prefix::-3} - if [[ $(( ${cluster_size} % 16 )) -eq 0 && $( ${pod_prefix: -1} ) != ":" ]]; then # should not be upper byte for this size CIDR - echo "ERROR! Trimmed address part of CIDR (${pod_prefix}) is still too large for /${cluster_size}" - exit 1 - fi - fi - # Add in zeros to pad 16 bits at a time, up to the padding needed, which is - # need_zero_pads - num_colons. - while [ ${num_colons} -lt ${need_zero_pads} ]; do - pod_prefix+="0:" - ((num_colons++)) - done - pod_prefixes+=( "${pod_prefix}" ) - fi -done - -DIND_IMAGE="${DIND_IMAGE:-}" -BUILD_KUBEADM="${BUILD_KUBEADM:-}" -BUILD_HYPERKUBE="${BUILD_HYPERKUBE:-}" -if [[ ! -z ${DIND_K8S_BIN_DIR:-} ]]; then - BUILD_KUBEADM="" - BUILD_HYPERKUBE="" -fi -KUBEADM_SOURCE="${KUBEADM_SOURCE-}" -HYPERKUBE_SOURCE="${HYPERKUBE_SOURCE-}" -NUM_NODES=${NUM_NODES:-2} -EXTRA_PORTS="${EXTRA_PORTS:-}" -LOCAL_KUBECTL_VERSION=${LOCAL_KUBECTL_VERSION:-} -KUBECTL_DIR="${KUBECTL_DIR:-${HOME}/.kubeadm-dind-cluster}" -DASHBOARD_URL="${DASHBOARD_URL:-https://rawgit.com/kubernetes/dashboard/bfab10151f012d1acc5dfb1979f3172e2400aa3c/src/deploy/kubernetes-dashboard.yaml}" -SKIP_SNAPSHOT="${SKIP_SNAPSHOT:-}" -E2E_REPORT_DIR="${E2E_REPORT_DIR:-}" -DIND_NO_PARALLEL_E2E="${DIND_NO_PARALLEL_E2E:-}" -DNS_SERVICE="${DNS_SERVICE:-kube-dns}" -DIND_STORAGE_DRIVER="${DIND_STORAGE_DRIVER:-overlay2}" - -DIND_CA_CERT_URL="${DIND_CA_CERT_URL:-}" -DIND_PROPAGATE_HTTP_PROXY="${DIND_PROPAGATE_HTTP_PROXY:-}" -DIND_HTTP_PROXY="${DIND_HTTP_PROXY:-}" -DIND_HTTPS_PROXY="${DIND_HTTPS_PROXY:-}" -DIND_NO_PROXY="${DIND_NO_PROXY:-}" - -DIND_DAEMON_JSON_FILE="${DIND_DAEMON_JSON_FILE:-/etc/docker/daemon.json}" # can be set to /dev/null -DIND_REGISTRY_MIRROR="${DIND_REGISTRY_MIRROR:-}" # plain string format -DIND_INSECURE_REGISTRIES="${DIND_INSECURE_REGISTRIES:-}" # json list format - -FEATURE_GATES="${FEATURE_GATES:-MountPropagation=true}" -# you can set special value 'none' not to set any kubelet's feature gates. -KUBELET_FEATURE_GATES="${KUBELET_FEATURE_GATES:-MountPropagation=true,DynamicKubeletConfig=true}" - -if [[ ! ${LOCAL_KUBECTL_VERSION:-} && ${DIND_IMAGE:-} =~ :(v[0-9]+\.[0-9]+)$ ]]; then - LOCAL_KUBECTL_VERSION="${BASH_REMATCH[1]}" -fi - -ENABLE_CEPH="${ENABLE_CEPH:-}" - -DIND_CRI="${DIND_CRI:-docker}" -case "${DIND_CRI}" in - docker) - CRI_SOCKET=/var/run/dockershim.sock - ;; - containerd) - CRI_SOCKET=/var/run/containerd/containerd.sock - ;; - *) - echo >&2 "Bad DIND_CRI. Please specify 'docker' or 'containerd'" - ;; -esac - -# TODO: Test multi-cluster for IPv6, before enabling -if [[ "${DIND_LABEL}" != "${DEFAULT_DIND_LABEL}" && "${IP_MODE}" == 'dual-stack' ]]; then - echo "Multiple parallel clusters currently not supported for dual-stack mode" >&2 - exit 1 -fi - -# not configurable for now, would need to setup context for kubectl _inside_ the cluster -readonly INTERNAL_APISERVER_PORT=8080 - -function dind::need-source { - if [[ ! -f cluster/kubectl.sh ]]; then - echo "$0 must be called from the Kubernetes repository root directory" 1>&2 - exit 1 - fi -} - -build_tools_dir="build" -use_k8s_source=y -if [[ ! ${BUILD_KUBEADM} && ! ${BUILD_HYPERKUBE} ]]; then - use_k8s_source= -fi -if [[ ${use_k8s_source} ]]; then - dind::need-source - kubectl=cluster/kubectl.sh - if [[ ! -f ${build_tools_dir}/common.sh ]]; then - build_tools_dir="build-tools" - fi -else - if [[ ! ${LOCAL_KUBECTL_VERSION:-} ]] && ! hash kubectl 2>/dev/null; then - echo "You need kubectl binary in your PATH to use prebuilt DIND image" 1>&2 - exit 1 - fi - kubectl=kubectl -fi - -function dind::retry { - # based on retry function in hack/jenkins/ scripts in k8s source - for i in {1..10}; do - "$@" && return 0 || sleep ${i} - done - "$@" -} - -busybox_image="busybox:1.26.2" -e2e_base_image="golang:1.10.5" -sys_volume_args=() -build_volume_args=() - -function dind::set-build-volume-args { - if [ ${#build_volume_args[@]} -gt 0 ]; then - return 0 - fi - build_container_name= - if [ -n "${KUBEADM_DIND_LOCAL:-}" ]; then - build_volume_args=(-v "$PWD:/go/src/k8s.io/kubernetes") - else - build_container_name="$(KUBE_ROOT=${PWD} ETCD_HOST=${ETCD_HOST} && - . ${build_tools_dir}/common.sh && - kube::build::verify_prereqs >&2 && - echo "${KUBE_DATA_CONTAINER_NAME:-${KUBE_BUILD_DATA_CONTAINER_NAME}}")" - build_volume_args=(--volumes-from "${build_container_name}") - fi -} - -function dind::volume-exists { - local name="$1" - if docker volume inspect "${name}" >& /dev/null; then - return 0 - fi - return 1 -} - -function dind::create-volume { - local name="$1" - docker volume create --label "${DIND_LABEL}" --name "${name}" >/dev/null -} - -# We mount /boot and /lib/modules into the container -# below to in case some of the workloads need them. -# This includes virtlet, for instance. Also this may be -# useful in future if we want DIND nodes to pass -# preflight checks. -# Unfortunately we can't do this when using Mac Docker -# (unless a remote docker daemon on Linux is used) -# NB: there's no /boot on recent Mac dockers -function dind::prepare-sys-mounts { - if [[ ! ${using_linuxkit} ]]; then - sys_volume_args=() - if [[ -d /boot ]]; then - sys_volume_args+=(-v /boot:/boot) - fi - if [[ -d /lib/modules ]]; then - sys_volume_args+=(-v /lib/modules:/lib/modules) - fi - return 0 - fi - local dind_sys_vol_name - dind_sys_vol_name="kubeadm-dind-sys$( dind::cluster-suffix )" - if ! dind::volume-exists "$dind_sys_vol_name"; then - dind::step "Saving a copy of docker host's /lib/modules" - dind::create-volume "$dind_sys_vol_name" - # Use a dirty nsenter trick to fool Docker on Mac and grab system - # /lib/modules into sys.tar file on kubeadm-dind-sys volume. - local nsenter="nsenter --mount=/proc/1/ns/mnt --" - docker run \ - --rm \ - --privileged \ - -v "$dind_sys_vol_name":/dest \ - --pid=host \ - "${busybox_image}" \ - /bin/sh -c \ - "if ${nsenter} test -d /lib/modules; then ${nsenter} tar -C / -c lib/modules >/dest/sys.tar; fi" - fi - sys_volume_args=(-v "$dind_sys_vol_name":/dind-sys) -} - -tmp_containers=() - -function dind::cleanup { - if [ ${#tmp_containers[@]} -gt 0 ]; then - for name in "${tmp_containers[@]}"; do - docker rm -vf "${name}" 2>/dev/null - done - fi -} - -trap dind::cleanup EXIT - -function dind::check-image { - local name="$1" - if docker inspect --format 'x' "${name}" >&/dev/null; then - return 0 - else - return 1 - fi -} - -function dind::filter-make-output { - # these messages make output too long and make Travis CI choke - egrep -v --line-buffered 'I[0-9][0-9][0-9][0-9] .*(parse|conversion|defaulter|deepcopy)\.go:[0-9]+\]' -} - -function dind::run-build-command { - # this is like build/run.sh, but it doesn't rsync back the binaries, - # only the generated files. - local cmd=("$@") - ( - # The following is taken from build/run.sh and build/common.sh - # of Kubernetes source tree. It differs in - # --filter='+ /_output/dockerized/bin/**' - # being removed from rsync - . ${build_tools_dir}/common.sh - kube::build::verify_prereqs - kube::build::build_image - kube::build::run_build_command "$@" - - kube::log::status "Syncing out of container" - - kube::build::start_rsyncd_container - - local rsync_extra="" - if (( ${KUBE_VERBOSE} >= 6 )); then - rsync_extra="-iv" - fi - - # The filter syntax for rsync is a little obscure. It filters on files and - # directories. If you don't go in to a directory you won't find any files - # there. Rules are evaluated in order. The last two rules are a little - # magic. '+ */' says to go in to every directory and '- /**' says to ignore - # any file or directory that isn't already specifically allowed. - # - # We are looking to copy out all of the built binaries along with various - # generated files. - kube::build::rsync \ - --filter='- /vendor/' \ - --filter='- /_temp/' \ - --filter='+ zz_generated.*' \ - --filter='+ generated.proto' \ - --filter='+ *.pb.go' \ - --filter='+ types.go' \ - --filter='+ */' \ - --filter='- /**' \ - "rsync://k8s@${KUBE_RSYNC_ADDR}/k8s/" "${KUBE_ROOT}" - - kube::build::stop_rsyncd_container - ) -} - -function dind::make-for-linux { - local copy="$1" - shift - dind::step "Building binaries:" "$*" - if [ -n "${KUBEADM_DIND_LOCAL:-}" ]; then - dind::step "+ make WHAT=\"$*\"" - make WHAT="$*" 2>&1 | dind::filter-make-output - elif [ "${copy}" = "y" ]; then - dind::step "+ ${build_tools_dir}/run.sh make WHAT=\"$*\"" - "${build_tools_dir}/run.sh" make WHAT="$*" 2>&1 | dind::filter-make-output - else - dind::step "+ [using the build container] make WHAT=\"$*\"" - dind::run-build-command make WHAT="$*" 2>&1 | dind::filter-make-output - fi -} - -function dind::check-binary { - local filename="$1" - local dockerized="_output/dockerized/bin/linux/amd64/${filename}" - local plain="_output/local/bin/linux/amd64/${filename}" - dind::set-build-volume-args - # FIXME: don't hardcode amd64 arch - if [ -n "${KUBEADM_DIND_LOCAL:-${force_local:-}}" ]; then - if [ -f "${dockerized}" -o -f "${plain}" ]; then - return 0 - fi - elif docker run --rm "${build_volume_args[@]}" \ - "${busybox_image}" \ - test -f "/go/src/k8s.io/kubernetes/${dockerized}" >&/dev/null; then - return 0 - fi - return 1 -} - -function dind::ensure-downloaded-kubectl { - local kubectl_url - local kubectl_sha1 - local kubectl_sha1_linux - local kubectl_sha1_darwin - local kubectl_link - local kubectl_os - local full_kubectl_version - - case "${LOCAL_KUBECTL_VERSION}" in - v1.9) - full_kubectl_version=v1.9.11 - kubectl_sha1_linux=3dedd41a077be12668c2ee322958dd3e9e554861 - kubectl_sha1_darwin=3b436f60ff0399b3f521d549044513358487ce20 - ;; - v1.10) - full_kubectl_version=v1.10.9 - kubectl_sha1_linux=bf3914630fe45b4f9ec1bc5e56f10fb30047f958 - kubectl_sha1_darwin=6833874e0b24fa1857925423e4dc8aeaa322b7b3 - ;; - v1.11) - full_kubectl_version=v1.11.3 - kubectl_sha1_linux=5a336ea470a0053b1893fda89a538b6197566137 - kubectl_sha1_darwin=4f519f6c225f2dd663a26c5c2390849324a577cb - ;; - v1.12) - full_kubectl_version=v1.12.1 - kubectl_sha1_linux=2135356e8e205816829f612062e1b5b4e1c81a17 - kubectl_sha1_darwin=a99adb19c1fa0334ab9be37b3918e5de84436acd - ;; - "") - return 0 - ;; - *) - echo "Invalid kubectl version" >&2 - exit 1 - esac - - export PATH="${KUBECTL_DIR}:$PATH" - - if [ $(uname) = Darwin ]; then - kubectl_sha1="${kubectl_sha1_darwin}" - kubectl_os=darwin - else - kubectl_sha1="${kubectl_sha1_linux}" - kubectl_os=linux - fi - local link_target="kubectl-${full_kubectl_version}" - local link_name="${KUBECTL_DIR}"/kubectl - if [[ -h "${link_name}" && "$(readlink "${link_name}")" = "${link_target}" ]]; then - return 0 - fi - - local path="${KUBECTL_DIR}/${link_target}" - if [[ ! -f "${path}" ]]; then - mkdir -p "${KUBECTL_DIR}" - curl -sSLo "${path}" "https://storage.googleapis.com/kubernetes-release/release/${full_kubectl_version}/bin/${kubectl_os}/amd64/kubectl" - echo "${kubectl_sha1} ${path}" | sha1sum -c - chmod +x "${path}" - fi - - ln -fs "${link_target}" "${KUBECTL_DIR}/kubectl" -} - -function dind::ensure-kubectl { - if [[ ! ${use_k8s_source} ]]; then - # already checked on startup - dind::ensure-downloaded-kubectl - return 0 - fi - if [ $(uname) = Darwin ]; then - if [ ! -f _output/local/bin/darwin/amd64/kubectl ]; then - dind::step "Building kubectl" - dind::step "+ make WHAT=cmd/kubectl" - make WHAT=cmd/kubectl 2>&1 | dind::filter-make-output - fi - elif ! force_local=y dind::check-binary kubectl; then - dind::make-for-linux y cmd/kubectl - fi -} - -function dind::ensure-binaries { - local -a to_build=() - for name in "$@"; do - if ! dind::check-binary "$(basename "${name}")"; then - to_build+=("${name}") - fi - done - if [ "${#to_build[@]}" -gt 0 ]; then - dind::make-for-linux n "${to_build[@]}" - fi - return 0 -} - -# dind::ensure-network creates the management network for the cluster. For IPv4 -# only it will have the management network CIDR. For IPv6 only, it will have -# the IPv6 management network CIDR and the NAT64 V4 mapping network CIDR. For -# dual stack, it will have the IPv4 and IPv6 management CIDRs. Each of the -# management networks (not the NAT64 network) will have a gateway specified. -# -function dind::ensure-network { - if ! docker network inspect $(dind::net-name) >&/dev/null; then - local -a args - for cidr in "${mgmt_net_cidrs[@]}"; do - if [[ $( dind::family-for ${cidr} ) = "ipv6" ]]; then - args+=(--ipv6) - fi - args+=(--subnet="${cidr}") - local gw=$( dind::make-ip-from-cidr ${cidr} 1 ) - args+=(--gateway="${gw}") - done - if [[ ${IP_MODE} = "ipv6" ]]; then - # Need second network for NAT64 V4 mapping network - args+=(--subnet=${NAT64_V4_SUBNET_PREFIX}.0.0/16) - fi - docker network create ${args[@]} $(dind::net-name) >/dev/null - fi -} - -function dind::ensure-volume { - local reuse_volume= - if [[ $1 = -r ]]; then - reuse_volume=1 - shift - fi - local name="$1" - if dind::volume-exists "${name}"; then - if [[ ! ${reuse_volume} ]]; then - docker volume rm "${name}" >/dev/null - fi - fi - dind::create-volume "${name}" -} - -function dind::ensure-dns { - if [[ ${IP_MODE} = "ipv6" ]]; then - local dns64_name="bind9$( dind::cluster-suffix )" - if ! docker inspect ${dns64_name} >&/dev/null; then - local force_dns64_for="" - if [[ ! ${DIND_ALLOW_AAAA_USE} ]]; then - # Normally, if have an AAAA record, it is used. This clause tells - # bind9 to do ignore AAAA records for the specified networks - # and/or addresses and lookup A records and synthesize new AAAA - # records. In this case, we select "any" networks that have AAAA - # records meaning we ALWAYS use A records and do NAT64. - force_dns64_for="exclude { any; };" - fi - read -r -d '' bind9_conf </named.conf && named -c /named.conf -g -u named' >/dev/null - ipv4_addr="$(docker exec ${dns64_name} ip addr list eth0 | grep "inet" | awk '$1 == "inet" {print $2}')" - docker exec ${dns64_name} ip addr del ${ipv4_addr} dev eth0 - docker exec ${dns64_name} ip -6 route add ${DNS64_PREFIX_CIDR} via ${LOCAL_NAT64_SERVER} - fi - fi -} - -function dind::ensure-nat { - if [[ ${IP_MODE} = "ipv6" ]]; then - local nat64_name="tayga$( dind::cluster-suffix )" - if ! docker ps | grep ${nat64_name} >&/dev/null; then - docker run -d --name ${nat64_name} --hostname ${nat64_name} --net "$(dind::net-name)" --label "dind-support$( dind::cluster-suffix )" \ - --sysctl net.ipv6.conf.all.disable_ipv6=0 --sysctl net.ipv6.conf.all.forwarding=1 \ - --privileged=true --ip ${NAT64_V4_SUBNET_PREFIX}.0.200 --ip6 ${LOCAL_NAT64_SERVER} --dns ${REMOTE_DNS64_V4SERVER} --dns ${dns_server} \ - -e TAYGA_CONF_PREFIX=${DNS64_PREFIX_CIDR} -e TAYGA_CONF_IPV4_ADDR=${NAT64_V4_SUBNET_PREFIX}.0.200 \ - -e TAYGA_CONF_DYNAMIC_POOL=${NAT64_V4_SUBNET_PREFIX}.0.128/25 danehans/tayga:latest >/dev/null - # Need to check/create, as "clean" may remove route - local route="$(ip route | egrep "^${NAT64_V4_SUBNET_PREFIX}.0.128/25")" - if [[ -z "${route}" ]]; then - docker run --net=host --rm --privileged ${busybox_image} ip route add ${NAT64_V4_SUBNET_PREFIX}.0.128/25 via ${NAT64_V4_SUBNET_PREFIX}.0.200 - fi - fi - fi -} - -function dind::run { - local reuse_volume= - if [[ $1 = -r ]]; then - reuse_volume="-r" - shift - fi - local container_name="${1:-}" - local node_id=${2:-0} - local portforward="${3:-}" - if [[ $# -gt 3 ]]; then - shift 3 - else - shift $# - fi - - local -a opts=("$@") - local ip_mode="--ip" - for cidr in "${mgmt_net_cidrs[@]}"; do - if [[ $( dind::family-for ${cidr} ) = "ipv6" ]]; then - ip_mode="--ip6" - fi - opts+=("${ip_mode}" "$( dind::make-ip-from-cidr ${cidr} $((${node_id}+1)) )") - done - opts+=("$@") - - local -a args=("systemd.setenv=CNI_PLUGIN=${CNI_PLUGIN}") - args+=("systemd.setenv=IP_MODE=${IP_MODE}") - args+=("systemd.setenv=DIND_STORAGE_DRIVER=${DIND_STORAGE_DRIVER}") - args+=("systemd.setenv=DIND_CRI=${DIND_CRI}") - - if [[ ${IP_MODE} != "ipv4" ]]; then - opts+=(--sysctl net.ipv6.conf.all.disable_ipv6=0) - opts+=(--sysctl net.ipv6.conf.all.forwarding=1) - fi - - if [[ ${IP_MODE} = "ipv6" ]]; then - opts+=(--dns ${dns_server}) - args+=("systemd.setenv=DNS64_PREFIX_CIDR=${DNS64_PREFIX_CIDR}") - args+=("systemd.setenv=LOCAL_NAT64_SERVER=${LOCAL_NAT64_SERVER}") - fi - - declare -a pod_nets - local i=0 - if [[ ${IP_MODE} = "ipv4" || ${IP_MODE} = "dual-stack" ]]; then - pod_nets+=("${pod_prefixes[$i]}${node_id}") - i=$((i+1)) - fi - if [[ ${IP_MODE} = "ipv6" || ${IP_MODE} = "dual-stack" ]]; then - # For prefix, if node ID will be in the upper byte, push it over - if [[ $((${pod_sizes[$i]} % 16)) -ne 0 ]]; then - n_id=$(printf "%02x00\n" "${node_id}") - else - if [[ "${pod_prefixes[$i]: -1}" = ":" ]]; then - n_id=$(printf "%x\n" "${node_id}") - else - n_id=$(printf "%02x\n" "${node_id}") # In lower byte, so ensure two chars - fi - fi - pod_nets+=("${pod_prefixes[$i]}${n_id}") - fi - - args+=("systemd.setenv=POD_NET_PREFIX=\"${pod_nets[0]}\"") - args+=("systemd.setenv=POD_NET_SIZE=\"${pod_sizes[0]}\"") - args+=("systemd.setenv=POD_NET2_PREFIX=\"${pod_nets[1]:-}\"") - args+=("systemd.setenv=POD_NET2_SIZE=\"${pod_sizes[1]:-}\"") - args+=("systemd.setenv=SERVICE_NET_MODE=${SERVICE_NET_MODE}") - args+=("systemd.setenv=USE_HAIRPIN=${USE_HAIRPIN}") - args+=("systemd.setenv=DNS_SVC_IP=${DNS_SVC_IP}") - args+=("systemd.setenv=DNS_SERVICE=${DNS_SERVICE}") - if [[ ! "${container_name}" ]]; then - echo >&2 "Must specify container name" - exit 1 - fi - - # remove any previously created containers with the same name - docker rm -vf "${container_name}" >&/dev/null || true - - if [[ "${portforward}" ]]; then - IFS=';' read -ra array <<< "${portforward}" - for element in "${array[@]}"; do - opts+=(-p "${element}") - done - fi - - opts+=(${sys_volume_args[@]+"${sys_volume_args[@]}"}) - - dind::step "Starting DIND container:" "${container_name}" - - if [[ ! -z ${DIND_K8S_BIN_DIR:-} ]]; then - opts+=(-v ${DIND_K8S_BIN_DIR}:/k8s) - fi - if [[ ! ${using_linuxkit} ]]; then - opts+=(-v /boot:/boot -v /lib/modules:/lib/modules) - fi - - if [[ ${ENABLE_CEPH} ]]; then - opts+=(-v /dev:/dev - -v /sys/bus:/sys/bus - -v /var/run/docker.sock:/opt/outer-docker.sock) - fi - - local volume_name="kubeadm-dind-${container_name}" - dind::ensure-network - dind::ensure-volume ${reuse_volume} "${volume_name}" - dind::ensure-nat - dind::ensure-dns - - # TODO: create named volume for binaries and mount it to /k8s - # in case of the source build - - # Start the new container. - docker run \ - -e IP_MODE="${IP_MODE}" \ - -e KUBEADM_SOURCE="${KUBEADM_SOURCE}" \ - -e HYPERKUBE_SOURCE="${HYPERKUBE_SOURCE}" \ - -d --privileged \ - --net "$(dind::net-name)" \ - --name "${container_name}" \ - --hostname "${container_name}" \ - -l "${DIND_LABEL}" \ - -v "${volume_name}:/dind" \ - ${opts[@]+"${opts[@]}"} \ - "${DIND_IMAGE}" \ - ${args[@]+"${args[@]}"} -} - -function dind::kubeadm { - local container_id="$1" - shift - dind::step "Running kubeadm:" "$*" - status=0 - # See image/bare/wrapkubeadm. - # Capturing output is necessary to grab flags for 'kubeadm join' - local -a env=(-e KUBELET_FEATURE_GATES="${KUBELET_FEATURE_GATES}" - -e DIND_CRI="${DIND_CRI}") - if ! docker exec "${env[@]}" "${container_id}" /usr/local/bin/wrapkubeadm "$@" 2>&1 | tee >(cat 1>&2); then - echo "*** kubeadm failed" >&2 - return 1 - fi - return ${status} -} - -# function dind::bare { -# local container_name="${1:-}" -# if [[ ! "${container_name}" ]]; then -# echo >&2 "Must specify container name" -# exit 1 -# fi -# shift -# run_opts=(${@+"$@"}) -# dind::run "${container_name}" -# } - -function dind::configure-kubectl { - dind::step "Setting cluster config" - local host="$(dind::localhost)" - if [[ -z "$using_local_linuxdocker" ]]; then - host="127.0.0.1" - fi - local context_name cluster_name - context_name="$(dind::context-name)" - cluster_name="$(dind::context-name)" - "${kubectl}" config set-cluster "$cluster_name" \ - --server="http://${host}:$(dind::apiserver-port)" \ - --insecure-skip-tls-verify=true - "${kubectl}" config set-context "$context_name" --cluster="$cluster_name" - if [[ ${DIND_LABEL} = ${DEFAULT_DIND_LABEL} ]]; then - # Single cluster mode - "${kubectl}" config use-context "$context_name" - fi -} - -force_make_binaries= -function dind::set-master-opts { - master_opts=() - if [[ ${BUILD_KUBEADM} || ${BUILD_HYPERKUBE} ]]; then - # share binaries pulled from the build container between nodes - local dind_k8s_bin_vol_name - dind_k8s_bin_vol_name="dind-k8s-binaries$(dind::cluster-suffix)" - dind::ensure-volume -r "${dind_k8s_bin_vol_name}" - dind::set-build-volume-args - master_opts+=("${build_volume_args[@]}" -v "${dind_k8s_bin_vol_name}:/k8s") - local -a bins - if [[ ${BUILD_KUBEADM} ]]; then - master_opts+=(-e KUBEADM_SOURCE=build://) - bins+=(cmd/kubeadm) - else - master_opts+=(-e ${KUBEADM_SOURCE}) - fi - if [[ ${BUILD_HYPERKUBE} ]]; then - master_opts+=(-e HYPERKUBE_SOURCE=build://) - bins+=(cmd/hyperkube) - fi - if [[ ${force_make_binaries} ]]; then - dind::make-for-linux n "${bins[@]}" - else - dind::ensure-binaries "${bins[@]}" - fi - fi - if [[ ${MASTER_EXTRA_OPTS:-} ]]; then - master_opts+=( ${MASTER_EXTRA_OPTS} ) - fi -} - -function dind::ensure-dashboard-clusterrolebinding { - local ctx - ctx="$(dind::context-name)" - # 'create' may cause etcd timeout, yet create the clusterrolebinding. - # So use 'apply' to actually create it - "${kubectl}" --context "$ctx" create clusterrolebinding add-on-cluster-admin \ - --clusterrole=cluster-admin \ - --serviceaccount=kube-system:default \ - -o json --dry-run | - docker exec -i "$(dind::master-name)" jq '.apiVersion="rbac.authorization.k8s.io/v1beta1"|.kind|="ClusterRoleBinding"' | - "${kubectl}" --context "$ctx" apply -f - -} - -function dind::deploy-dashboard { - dind::step "Deploying k8s dashboard" - dind::retry "${kubectl}" --context "$(dind::context-name)" apply -f "${DASHBOARD_URL}" - # https://kubernetes-io-vnext-staging.netlify.com/docs/admin/authorization/rbac/#service-account-permissions - # Thanks @liggitt for the hint - dind::retry dind::ensure-dashboard-clusterrolebinding -} - -function dind::kubeadm-version { - if [[ ${use_k8s_source} ]]; then - (cluster/kubectl.sh version --short 2>/dev/null || true) | - grep Client | - sed 's/^.*: v\([0-9.]*\).*/\1/' - else - docker exec "$(dind::master-name)" \ - /bin/bash -c 'kubeadm version -o json | jq -r .clientVersion.gitVersion' | - sed 's/^v\([0-9.]*\).*/\1/' - fi -} - -function dind::kubeadm-version-at-least { - local major="${1}" - local minor="${2}" - if [[ ! ( $(dind::kubeadm-version) =~ ^([0-9]+)\.([0-9]+) ) ]]; then - echo >&2 "WARNING: can't parse kubeadm version: $(dind::kubeadm-version)" - return 1 - fi - local act_major="${BASH_REMATCH[1]}" - local act_minor="${BASH_REMATCH[2]}" - if [[ ${act_major} -gt ${major} ]]; then - return 0 - fi - if [[ ${act_major} -lt ${major} ]]; then - return 1 - fi - if [[ ${act_minor} -ge ${minor} ]]; then - return 0 - fi - return 1 -} - -function dind::verify-image-compatibility { - # We can't tell in advance, if the image selected supports dual-stack, - # but will do the next best thing, and check as soon as start up kube-master - local master_name=$1 - if [[ ${IP_MODE} = "dual-stack" ]]; then - local dual_stack_support="$(docker exec ${master_name} cat /node-info 2>/dev/null | grep "dual-stack-support" | wc -l)" - if [[ ${dual_stack_support} -eq 0 ]]; then - echo "ERROR! DinD image (${DIND_IMAGE}) does not support dual-stack mode - aborting!" - dind::remove-images "${DIND_LABEL}" - exit 1 - fi - fi -} - -function dind::check-dns-service-type { - if [[ ${DNS_SERVICE} = "kube-dns" ]] && dind::kubeadm-version-at-least 1 14; then - echo >&2 "WARNING: for 1.13+, only coredns can be used as the DNS service" - DNS_SERVICE="coredns" - fi -} - -function dind::init { - local -a opts - dind::set-master-opts - local local_host master_name container_id - master_name="$(dind::master-name)" - local_host="$( dind::localhost )" - container_id=$(dind::run "${master_name}" 1 "${local_host}:$(dind::apiserver-port):${INTERNAL_APISERVER_PORT}" ${master_opts[@]+"${master_opts[@]}"}) - - dind::verify-image-compatibility ${master_name} - - # FIXME: I tried using custom tokens with 'kubeadm ex token create' but join failed with: - # 'failed to parse response as JWS object [square/go-jose: compact JWS format must have three parts]' - # So we just pick the line from 'kubeadm init' output - # Using a template file in the image (based on version) to build a kubeadm.conf file and to customize - # it based on CNI plugin, IP mode, and environment settings. User can add additional - # customizations to template and then rebuild the image used (build/build-local.sh). - local pod_subnet_disable="# " - # TODO: May want to specify each of the plugins that require --pod-network-cidr - if [[ ${CNI_PLUGIN} != "bridge" && ${CNI_PLUGIN} != "ptp" ]]; then - pod_subnet_disable="" - fi - local bind_address="0.0.0.0" - if [[ ${SERVICE_NET_MODE} = "ipv6" ]]; then - bind_address="::" - fi - dind::proxy "$master_name" - dind::custom-docker-opts "$master_name" - - # HACK: Indicating mode, so that wrapkubeadm will not set a cluster CIDR for kube-proxy - # in IPv6 (only) mode. - if [[ ${SERVICE_NET_MODE} = "ipv6" ]]; then - docker exec --privileged -i "$master_name" touch /v6-mode - fi - - feature_gates="{CoreDNS: false}" - if [[ ${DNS_SERVICE} == "coredns" ]]; then - feature_gates="{CoreDNS: true}" - fi - - kubeadm_version="$(dind::kubeadm-version)" - case "${kubeadm_version}" in - 1\.9\.* | 1\.10\.*) - template="1.10" - ;; - 1\.11\.*) - template="1.11" - ;; - 1\.12\.*) - template="1.12" - ;; - 1\.13\.*) - template="1.13" - ;; - *) # Includes master branch - # Will make a separate template if/when it becomes incompatible - template="1.13" - # CoreDNS can no longer be switched off - feature_gates="{}" - ;; - esac - dind::check-dns-service-type - - component_feature_gates="" - if [ "${FEATURE_GATES}" != "none" ]; then - component_feature_gates="feature-gates: \\\"${FEATURE_GATES}\\\"" - fi - - apiserver_extra_args="" - for e in $(set -o posix ; set | grep -E "^APISERVER_[a-z_]+=" | cut -d'=' -f 1); do - opt_name=$(echo ${e#APISERVER_} | sed 's/_/-/g') - apiserver_extra_args+=" ${opt_name}: \\\"$(eval echo \$$e)\\\"\\n" - done - - controller_manager_extra_args="" - for e in $(set -o posix ; set | grep -E "^CONTROLLER_MANAGER_[a-z_]+=" | cut -d'=' -f 1); do - opt_name=$(echo ${e#CONTROLLER_MANAGER_} | sed 's/_/-/g') - controller_manager_extra_args+=" ${opt_name}: \\\"$(eval echo \$$e)\\\"\\n" - done - - scheduler_extra_args="" - for e in $(set -o posix ; set | grep -E "^SCHEDULER_[a-z_]+=" | cut -d'=' -f 1); do - opt_name=$(echo ${e#SCHEDULER_} | sed 's/_/-/g') - scheduler_extra_args+=" ${opt_name}: \\\"$(eval echo \$$e)\\\"\\n" - done - - local mgmt_cidr=${mgmt_net_cidrs[0]} - if [[ ${IP_MODE} = "dual-stack" && ${SERVICE_NET_MODE} = "ipv6" ]]; then - mgmt_cidr=${mgmt_net_cidrs[1]} - fi - local master_ip=$( dind::make-ip-from-cidr ${mgmt_cidr} 2 ) - docker exec -i "$master_name" bash < /etc/kubeadm.conf -EOF - init_args=(--config /etc/kubeadm.conf) - # required when building from source - if [[ ${BUILD_KUBEADM} || ${BUILD_HYPERKUBE} ]]; then - docker exec "$master_name" mount --make-shared /k8s - fi - kubeadm_join_flags="$(dind::kubeadm "${container_id}" init "${init_args[@]}" --ignore-preflight-errors=all "$@" | grep '^ *kubeadm join' | sed 's/^ *kubeadm join //')" - dind::configure-kubectl - dind::start-port-forwarder -} - -function dind::create-node-container { - local reuse_volume next_node_index node_name - reuse_volume='' - if [[ ${1:-} = -r ]]; then - reuse_volume="-r" - shift - fi - # if there's just one node currently, it's master, thus we need to use - # kube-node-1 hostname, if there are two nodes, we should pick - # kube-node-2 and so on - next_node_index=${1:-$(docker ps -q --filter=label="${DIND_LABEL}" | wc -l | sed 's/^ *//g')} - local -a opts - if [[ ${BUILD_KUBEADM} || ${BUILD_HYPERKUBE} ]]; then - opts+=(-v "dind-k8s-binaries$(dind::cluster-suffix)":/k8s) - if [[ ${BUILD_KUBEADM} ]]; then - opts+=(-e KUBEADM_SOURCE=build://) - fi - if [[ ${BUILD_HYPERKUBE} ]]; then - opts+=(-e HYPERKUBE_SOURCE=build://) - fi - fi - node_name="$(dind::node-name ${next_node_index})" - dind::run ${reuse_volume} "$node_name" $((next_node_index + 1)) "${EXTRA_PORTS}" ${opts[@]+"${opts[@]}"} -} - -function dind::join { - local container_id="$1" - shift - dind::proxy "${container_id}" - dind::custom-docker-opts "${container_id}" - local -a join_opts=(--ignore-preflight-errors=all - --cri-socket="${CRI_SOCKET}") - dind::kubeadm "${container_id}" join "${join_opts[@]}" "$@" >/dev/null -} - -function dind::escape-e2e-name { - sed 's/[]\$*.^()[]/\\&/g; s/\s\+/\\s+/g' <<< "$1" | tr -d '\n' -} - -function dind::accelerate-kube-dns { - if [[ ${DNS_SERVICE} == "kube-dns" ]]; then - dind::step "Patching kube-dns deployment to make it start faster" - # Could do this on the host, too, but we don't want to require jq here - # TODO: do this in wrapkubeadm - docker exec "$(dind::master-name)" /bin/bash -c \ - "kubectl get deployment kube-dns -n kube-system -o json | jq '.spec.template.spec.containers[0].readinessProbe.initialDelaySeconds = 3|.spec.template.spec.containers[0].readinessProbe.periodSeconds = 3' | kubectl apply --force -f -" - fi -} - -function dind::component-ready { - local label="$1" - local out - if ! out="$("${kubectl}" --context "$(dind::context-name)" get pod -l "${label}" -n kube-system \ - -o jsonpath='{ .items[*].status.conditions[?(@.type == "Ready")].status }' 2>/dev/null)"; then - return 1 - fi - if ! grep -v False <<<"${out}" | grep -q True; then - return 1 - fi - return 0 -} - -function dind::kill-failed-pods { - local pods ctx - ctx="$(dind::context-name)" - # workaround for https://github.com/kubernetes/kubernetes/issues/36482 - if ! pods="$(kubectl --context "$ctx" get pod -n kube-system -o jsonpath='{ .items[?(@.status.phase == "Failed")].metadata.name }' 2>/dev/null)"; then - return - fi - for name in ${pods}; do - kubectl --context "$ctx" delete pod --now -n kube-system "${name}" >&/dev/null || true - done -} - -function dind::create-static-routes { - echo "Creating static routes for bridge/PTP plugin" - for ((i=0; i <= NUM_NODES; i++)); do - if [[ ${i} -eq 0 ]]; then - node="$(dind::master-name)" - else - node="$(dind::node-name $i)" - fi - for ((j=0; j <= NUM_NODES; j++)); do - if [[ ${i} -eq ${j} ]]; then - continue - fi - if [[ ${j} -eq 0 ]]; then - dest_node="$(dind::master-name)" - else - dest_node="$(dind::node-name $j)" - fi - id=$((${j}+1)) - if [[ ${IP_MODE} = "ipv4" || ${IP_MODE} = "dual-stack" ]]; then - # Assuming pod subnets will all be /24 - dest="${pod_prefixes[0]}${id}.0/24" - gw=`docker exec ${dest_node} ip addr show eth0 | grep -w inet | awk '{ print $2 }' | sed 's,/.*,,'` - docker exec "${node}" ip route add "${dest}" via "${gw}" - fi - if [[ ${IP_MODE} = "ipv6" || ${IP_MODE} = "dual-stack" ]]; then - local position=0 - if [[ ${IP_MODE} = "dual-stack" ]]; then - position=1 - fi - instance=$(printf "%02x" ${id}) - if [[ $((${pod_sizes[$position]} % 16)) -ne 0 ]]; then - instance+="00" # Move node ID to upper byte - fi - dest="${pod_prefixes[$position]}${instance}::/${pod_sizes[$position]}" - gw=`docker exec ${dest_node} ip addr show eth0 | grep -w inet6 | grep -i global | head -1 | awk '{ print $2 }' | sed 's,/.*,,'` - docker exec "${node}" ip route add "${dest}" via "${gw}" - fi - done - done -} - -# If we are allowing AAAA record use, then provide SNAT for IPv6 packets from -# node containers, and forward packets to bridge used for $(dind::net-name). -# This gives pods access to external IPv6 sites, when using IPv6 addresses. -function dind::setup_external_access_on_host { - if [[ ! ${DIND_ALLOW_AAAA_USE} ]]; then - return - fi - local main_if=`ip route | grep default | awk '{print $5}'` - dind::ip6tables-on-hostnet -t nat -A POSTROUTING -o $main_if -j MASQUERADE - if [[ ${IP_MODE} = "dual-stack" ]]; then - return - fi - local bridge_if=`ip route | grep ${NAT64_V4_SUBNET_PREFIX}.0.0 | awk '{print $3}'` - if [[ -n "$bridge_if" ]]; then - dind::ip6tables-on-hostnet -A FORWARD -i $bridge_if -j ACCEPT - else - echo "WARNING! No $(dind::net-name) bridge with NAT64 - unable to setup forwarding/SNAT" - fi -} - -# Remove ip6tables rules for SNAT and forwarding, if they exist. -function dind::remove_external_access_on_host { - if [[ ! ${DIND_ALLOW_AAAA_USE} ]]; then - return - fi - local have_rule - local main_if="$(ip route | grep default | awk '{print $5}')" - have_rule="$(dind::ip6tables-on-hostnet -S -t nat | grep "\-o $main_if" || true)" - if [[ -n "$have_rule" ]]; then - dind::ip6tables-on-hostnet -t nat -D POSTROUTING -o $main_if -j MASQUERADE - else - echo "Skipping delete of ip6tables rule for SNAT, as rule non-existent" - fi - - if [[ ${IP_MODE} = "dual-stack" ]]; then - return - fi - local bridge_if="$(ip route | grep ${NAT64_V4_SUBNET_PREFIX}.0.0 | awk '{print $3}')" - if [[ -n "$bridge_if" ]]; then - have_rule="$(dind::ip6tables-on-hostnet -S | grep "\-i $bridge_if" || true)" - if [[ -n "$have_rule" ]]; then - dind::ip6tables-on-hostnet -D FORWARD -i $bridge_if -j ACCEPT - else - echo "Skipping delete of ip6tables rule for forwarding, as rule non-existent" - fi - else - echo "Skipping delete of ip6tables rule for forwarding, as no bridge interface using NAT64" - fi -} - -function dind::ip6tables-on-hostnet { - local mod_path='/lib/modules' - docker run -v "${mod_path}:${mod_path}" --entrypoint /sbin/ip6tables --net=host --rm --privileged "${DIND_IMAGE}" "$@" -} - -function dind::wait-for-ready { - local app="kube-proxy" - if [[ ${CNI_PLUGIN} = "kube-router" ]]; then - app=kube-router - fi - dind::step "Waiting for ${app} and the nodes" - local app_ready - local nodes_ready - local n=3 - local ntries=200 - local ctx - ctx="$(dind::context-name)" - while true; do - dind::kill-failed-pods - if "${kubectl}" --context "$ctx" get nodes 2>/dev/null | grep -q NotReady; then - nodes_ready= - else - nodes_ready=y - fi - if dind::component-ready k8s-app=${app}; then - app_ready=y - else - app_ready= - fi - if [[ ${nodes_ready} && ${app_ready} ]]; then - if ((--n == 0)); then - echo "[done]" >&2 - break - fi - else - n=3 - fi - if ((--ntries == 0)); then - echo "Error waiting for ${app} and the nodes" >&2 - exit 1 - fi - echo -n "." >&2 - sleep 1 - done - - dind::step "Bringing up ${DNS_SERVICE} and kubernetes-dashboard" - # on Travis 'scale' sometimes fails with 'error: Scaling the resource failed with: etcdserver: request timed out; Current resource version 442' here - dind::retry "${kubectl}" --context "$ctx" scale deployment --replicas=1 -n kube-system ${DNS_SERVICE} - dind::retry "${kubectl}" --context "$ctx" scale deployment --replicas=1 -n kube-system kubernetes-dashboard - - ntries=200 - while ! dind::component-ready k8s-app=kube-dns || ! dind::component-ready app=kubernetes-dashboard; do - if ((--ntries == 0)); then - echo "Error bringing up ${DNS_SERVICE} and kubernetes-dashboard" >&2 - exit 1 - fi - echo -n "." >&2 - dind::kill-failed-pods - sleep 1 - done - echo "[done]" >&2 - - dind::retry "${kubectl}" --context "$ctx" get nodes >&2 - - local local_host - local_host="$( dind::localhost )" - dind::step "Access dashboard at:" "http://${local_host}:$(dind::apiserver-port)/api/v1/namespaces/kube-system/services/kubernetes-dashboard:/proxy" -} - -# dind::make-kube-router-yaml creates a temp file with contents of the configuration needed for the kube-router CNI -# plugin at a specific version, instead of using the publically available file, which uses the latest version. This -# allows us to control the version used. If/when updating, be sure to update the KUBE_ROUTER_VERSION env variable -# ensure the YAML contents below, reflect the configuration in: -# -# https://raw.githubusercontent.com/cloudnativelabs/kube-router/master/daemonset/kubeadm-kuberouter-all-feature.yaml -# -# FUTURE: May be able to remove this, if/when kube-router "latest" is stable, and use the public YAML file instead. -function dind::make-kube-router-yaml { - tmp_yaml=$(mktemp /tmp/kube-router-yaml.XXXXXX) - cat >${tmp_yaml} <&2 "*** Failed to start node container ${n}" - exit 1 - else - node_containers+=(${container_id}) - dind::step "Node container started:" ${n} - fi - done - dind::fix-mounts - status=0 - local -a pids - for ((n=1; n <= NUM_NODES; n++)); do - ( - dind::step "Joining node:" ${n} - container_id="${node_containers[${n}-1]}" - if ! dind::join ${container_id} ${kubeadm_join_flags}; then - echo >&2 "*** Failed to start node container ${n}" - exit 1 - else - dind::step "Node joined:" ${n} - fi - )& - pids[${n}]=$! - done - if ((NUM_NODES > 0)); then - for pid in ${pids[*]}; do - wait ${pid} - done - else - # FIXME: this may fail depending on k8s/kubeadm version - # FIXME: check for taint & retry if it's there - "${kubectl}" --context "$ctx" taint nodes $(dind::master-name) node-role.kubernetes.io/master- || true - fi - case "${CNI_PLUGIN}" in - bridge | ptp) - dind::create-static-routes - dind::setup_external_access_on_host - ;; - flannel) - # without --validate=false this will fail on older k8s versions - dind::retry "${kubectl}" --context "$ctx" apply --validate=false -f "https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel.yml?raw=true" - ;; - calico) - manifest_base=https://docs.projectcalico.org/${CALICO_VERSION:-v3.3}/getting-started/kubernetes/installation - dind::retry "${kubectl}" --context "$ctx" apply -f ${manifest_base}/hosted/etcd.yaml - if [ "${CALICO_VERSION:-v3.3}" != master ]; then - dind::retry "${kubectl}" --context "$ctx" apply -f ${manifest_base}/rbac.yaml - fi - dind::retry "${kubectl}" --context "$ctx" apply -f ${manifest_base}/hosted/calico.yaml - dind::retry "${kubectl}" --context "$ctx" apply -f ${manifest_base}/hosted/calicoctl.yaml - ;; - calico-kdd) - manifest_base=https://docs.projectcalico.org/${CALICO_VERSION:-v3.3}/getting-started/kubernetes/installation - dind::retry "${kubectl}" --context "$ctx" apply -f ${manifest_base}/hosted/rbac-kdd.yaml - dind::retry "${kubectl}" --context "$ctx" apply -f ${manifest_base}/hosted/kubernetes-datastore/calico-networking/1.7/calico.yaml - ;; - weave) - dind::retry "${kubectl}" --context "$ctx" apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(${kubectl} --context "$ctx" version | base64 | tr -d '\n')" - ;; - kube-router) - kube_router_config="$( dind::make-kube-router-yaml )" - dind::retry "${kubectl}" --context "$ctx" apply -f ${kube_router_config} - rm "${kube_router_config}" - dind::retry "${kubectl}" --context "$ctx" -n kube-system delete ds kube-proxy - docker run --privileged --net=host k8s.gcr.io/kube-proxy-amd64:v1.10.2 kube-proxy --cleanup - ;; - *) - echo "Unsupported CNI plugin '${CNI_PLUGIN}'" >&2 - ;; - esac - dind::deploy-dashboard - dind::accelerate-kube-dns - if [[ (${CNI_PLUGIN} != "bridge" && ${CNI_PLUGIN} != "ptp") || ${SKIP_SNAPSHOT} ]]; then - # This is especially important in case of Calico - - # the cluster will not recover after snapshotting - # (at least not after restarting from the snapshot) - # if Calico installation is interrupted - dind::wait-for-ready - fi - dind::step "Cluster Info" - echo "Network Mode: ${IP_MODE}" - echo "Cluster context: $( dind::context-name )" - echo "Cluster ID: ${CLUSTER_ID}" - echo "Management CIDR(s): ${mgmt_net_cidrs[@]}" - echo "Service CIDR/mode: ${SERVICE_CIDR}/${SERVICE_NET_MODE}" - echo "Pod CIDR(s): ${pod_net_cidrs[@]}" -} - -function dind::fix-mounts { - local node_name - for ((n=0; n <= NUM_NODES; n++)); do - node_name="$(dind::master-name)" - if ((n > 0)); then - node_name="$(dind::node-name $n)" - fi - docker exec "${node_name}" mount --make-shared /run - if [[ ! ${using_linuxkit} ]]; then - docker exec "${node_name}" mount --make-shared /lib/modules/ - fi - # required when building from source - if [[ ${BUILD_KUBEADM} || ${BUILD_HYPERKUBE} ]]; then - docker exec "${node_name}" mount --make-shared /k8s - fi - # docker exec "${node_name}" mount --make-shared /sys/kernel/debug - done -} - -function dind::snapshot_container { - local container_name="$1" - # we must pass DIND_CRI here because in case of containerd - # a special care must be taken to stop the containers during - # the snapshot - docker exec -e DIND_CRI="${DIND_CRI}" -i ${container_name} \ - /usr/local/bin/snapshot prepare - # remove the hidden *plnk directories - docker diff ${container_name} | grep -v plnk | docker exec -i ${container_name} /usr/local/bin/snapshot save -} - -function dind::snapshot { - dind::step "Taking snapshot of the cluster" - dind::snapshot_container "$(dind::master-name)" - for ((n=1; n <= NUM_NODES; n++)); do - dind::snapshot_container "$(dind::node-name $n)" - done - dind::wait-for-ready -} - -restore_cmd=restore -function dind::restore_container { - local container_id="$1" - docker exec ${container_id} /usr/local/bin/snapshot "${restore_cmd}" -} - -function dind::restore { - local apiserver_port local_host pid pids - dind::down - dind::check-dns-service-type - dind::step "Restoring containers" - dind::set-master-opts - local_host="$( dind::localhost )" - apiserver_port="$( dind::apiserver-port )" - for ((n=0; n <= NUM_NODES; n++)); do - ( - if [[ n -eq 0 ]]; then - dind::step "Restoring master container" - dind::restore_container "$(dind::run -r "$(dind::master-name)" 1 "${local_host}:${apiserver_port}:${INTERNAL_APISERVER_PORT}" ${master_opts[@]+"${master_opts[@]}"})" - dind::verify-image-compatibility "$(dind::master-name)" - dind::step "Master container restored" - else - dind::step "Restoring node container:" ${n} - if ! container_id="$(dind::create-node-container -r ${n})"; then - echo >&2 "*** Failed to start node container ${n}" - exit 1 - else - dind::restore_container "${container_id}" - dind::step "Node container restored:" ${n} - fi - fi - )& - pids[${n}]=$! - done - for pid in ${pids[*]}; do - wait ${pid} - done - if [[ ${CNI_PLUGIN} = "bridge" || ${CNI_PLUGIN} = "ptp" ]]; then - dind::create-static-routes - dind::setup_external_access_on_host - fi - dind::fix-mounts - # Recheck kubectl config. It's possible that the cluster was started - # on this docker from different host - dind::configure-kubectl - dind::start-port-forwarder - dind::wait-for-ready -} - -function dind::down { - dind::remove-images "${DIND_LABEL}" - if [[ ${CNI_PLUGIN} = "bridge" || ${CNI_PLUGIN} = "ptp" ]]; then - dind::remove_external_access_on_host - elif [[ "${CNI_PLUGIN}" = "kube-router" ]]; then - if [[ ${COMMAND} = "down" || ${COMMAND} = "clean" ]]; then - # FUTURE: Updated pinned version, after verifying operation - docker run --privileged --net=host cloudnativelabs/kube-router:${KUBE_ROUTER_VERSION} --cleanup-config - fi - fi -} - -function dind::apiserver-port { - # APISERVER_PORT is explicitely set - if [ -n "${APISERVER_PORT:-}" ] - then - echo "$APISERVER_PORT" - return - fi - - # Get the port from the master - local master port - master="$(dind::master-name)" - # 8080/tcp -> 127.0.0.1:8082 => 8082 - port="$( docker port "$master" 2>/dev/null | awk -F: "/^${INTERNAL_APISERVER_PORT}/{ print \$NF }" )" - if [ -n "$port" ] - then - APISERVER_PORT="$port" - echo "$APISERVER_PORT" - return - fi - - # get a random free port - APISERVER_PORT=0 - echo "$APISERVER_PORT" -} - -function dind::master-name { - echo "kube-master$( dind::cluster-suffix )" -} - -function dind::node-name { - local nr="$1" - echo "kube-node-${nr}$( dind::cluster-suffix )" -} - -function dind::context-name { - echo "dind$( dind::cluster-suffix )" -} - -function dind::remove-volumes { - # docker 1.13+: docker volume ls -q -f label="${DIND_LABEL}" - local nameRE - nameRE="^kubeadm-dind-(sys|kube-master|kube-node-[0-9]+)$(dind::cluster-suffix)$" - docker volume ls -q | (grep -E "$nameRE" || true) | while read -r volume_id; do - dind::step "Removing volume:" "${volume_id}" - docker volume rm "${volume_id}" - done -} - -function dind::remove-images { - local which=$1 - docker ps -a -q --filter=label="${which}" | while read container_id; do - dind::step "Removing container:" "${container_id}" - docker rm -fv "${container_id}" - done -} - -function dind::remove-cluster { - cluster_name="dind$(dind::cluster-suffix)" - if ${kubectl} config get-clusters | grep -qE "^${cluster_name}$"; then - dind::step "Removing cluster from config:" "${cluster_name}" - ${kubectl} config delete-cluster ${cluster_name} 2>/dev/null || true - fi -} - -function dind::remove-context { - context_name="$(dind::context-name)" - if ${kubectl} config get-contexts | grep -qE "${context_name}\\s"; then - dind::step "Removing context from config:" "${context_name}" - ${kubectl} config delete-context ${context_name} 2>/dev/null || true - fi -} - -function dind::start-port-forwarder { - local fwdr port - fwdr="${DIND_PORT_FORWARDER:-}" - - [ -n "$fwdr" ] || return 0 - - [ -x "$fwdr" ] || { - echo "'${fwdr}' is not executable." >&2 - return 1 - } - - port="$( dind::apiserver-port )" - dind::step "+ Setting up port-forwarding for :${port}" - "$fwdr" "$port" -} - -function dind::check-for-snapshot { - if ! dind::volume-exists "kubeadm-dind-$(dind::master-name)"; then - return 1 - fi - for ((n=1; n <= NUM_NODES; n++)); do - if ! dind::volume-exists "kubeadm-dind-$(dind::node-name ${n})"; then - return 1 - fi - done -} - -function dind::do-run-e2e { - local parallel="${1:-}" - local focus="${2:-}" - local skip="${3:-}" - local host="$(dind::localhost)" - if [[ -z "$using_local_linuxdocker" ]]; then - host="127.0.0.1" - fi - dind::need-source - local kubeapi test_args term= - local -a e2e_volume_opts=() - kubeapi="http://${host}:$(dind::apiserver-port)" - test_args="--host=${kubeapi}" - if [[ ${focus} ]]; then - test_args="--ginkgo.focus=${focus} ${test_args}" - fi - if [[ ${skip} ]]; then - test_args="--ginkgo.skip=${skip} ${test_args}" - fi - if [[ ${E2E_REPORT_DIR} ]]; then - test_args="--report-dir=/report ${test_args}" - e2e_volume_opts=(-v "${E2E_REPORT_DIR}:/report") - fi - dind::make-for-linux n "cmd/kubectl test/e2e/e2e.test vendor/github.com/onsi/ginkgo/ginkgo" - dind::step "Running e2e tests with args:" "${test_args}" - dind::set-build-volume-args - if [ -t 1 ] ; then - term="-it" - test_args="--ginkgo.noColor --num-nodes=2 ${test_args}" - fi - docker run \ - --rm ${term} \ - --net=host \ - "${build_volume_args[@]}" \ - -e KUBERNETES_PROVIDER=dind \ - -e KUBE_MASTER_IP="${kubeapi}" \ - -e KUBE_MASTER=local \ - -e KUBERNETES_CONFORMANCE_TEST=y \ - -e GINKGO_PARALLEL=${parallel} \ - ${e2e_volume_opts[@]+"${e2e_volume_opts[@]}"} \ - -w /go/src/k8s.io/kubernetes \ - "${e2e_base_image}" \ - bash -c "cluster/kubectl.sh config set-cluster dind --server='${kubeapi}' --insecure-skip-tls-verify=true && - cluster/kubectl.sh config set-context dind --cluster=dind && - cluster/kubectl.sh config use-context dind && - go run hack/e2e.go -- --v 6 --test --check-version-skew=false --test_args='${test_args}'" -} - -function dind::clean { - dind::down - dind::remove-images "dind-support$( dind::cluster-suffix )" - dind::remove-volumes - local net_name - net_name="$(dind::net-name)" - if docker network inspect "$net_name" >&/dev/null; then - docker network rm "$net_name" - fi - dind::remove-cluster - dind::remove-context -} - -function dind::copy-image { - local image="${2:-}" - local image_path="/tmp/save_${image//\//_}" - if [[ -f "${image_path}" ]]; then - rm -fr "${image_path}" - fi - docker save "${image}" -o "${image_path}" - docker ps -a -q --filter=label="${DIND_LABEL}" | while read container_id; do - cat "${image_path}" | docker exec -i "${container_id}" docker load - done - rm -fr "${image_path}" -} - -function dind::run-e2e { - local focus="${1:-}" - local skip="${2:-[Serial]}" - skip="$(dind::escape-e2e-name "${skip}")" - if [[ "$focus" ]]; then - focus="$(dind::escape-e2e-name "${focus}")" - else - focus="\[Conformance\]" - fi - local parallel=y - if [[ ${DIND_NO_PARALLEL_E2E} ]]; then - parallel= - fi - dind::do-run-e2e "${parallel}" "${focus}" "${skip}" -} - -function dind::run-e2e-serial { - local focus="${1:-}" - local skip="${2:-}" - skip="$(dind::escape-e2e-name "${skip}")" - dind::need-source - if [[ "$focus" ]]; then - focus="$(dind::escape-e2e-name "${focus}")" - else - focus="\[Serial\].*\[Conformance\]" - fi - dind::do-run-e2e n "${focus}" "${skip}" - # TBD: specify filter -} - -function dind::step { - local OPTS="" - if [ "$1" = "-n" ]; then - shift - OPTS+="-n" - fi - GREEN="$1" - shift - if [ -t 2 ] ; then - echo -e ${OPTS} "\x1B[97m* \x1B[92m${GREEN}\x1B[39m $*" 1>&2 - else - echo ${OPTS} "* ${GREEN} $*" 1>&2 - fi -} - -function dind::dump { - set +e - echo "*** Dumping cluster state ***" - for node in $(docker ps --format '{{.Names}}' --filter label="${DIND_LABEL}"); do - for service in kubelet.service dindnet.service criproxy.service dockershim.service; do - if docker exec "${node}" systemctl is-enabled "${service}" >&/dev/null; then - echo "@@@ service-${node}-${service}.log @@@" - docker exec "${node}" systemctl status "${service}" - docker exec "${node}" journalctl -xe -n all -u "${service}" - fi - done - echo "@@@ psaux-${node}.txt @@@" - docker exec "${node}" ps auxww - echo "@@@ dockerps-a-${node}.txt @@@" - docker exec "${node}" docker ps -a - echo "@@@ ip-a-${node}.txt @@@" - docker exec "${node}" ip a - echo "@@@ ip-r-${node}.txt @@@" - docker exec "${node}" ip r - done - local ctx master_name - master_name="$(dind::master-name)" - ctx="$(dind::context-name)" - docker exec "$master_name" kubectl get pods --all-namespaces \ - -o go-template='{{range $x := .items}}{{range $x.spec.containers}}{{$x.spec.nodeName}}{{" "}}{{$x.metadata.namespace}}{{" "}}{{$x.metadata.name}}{{" "}}{{.name}}{{"\n"}}{{end}}{{end}}' | - while read node ns pod container; do - echo "@@@ pod-${node}-${ns}-${pod}--${container}.log @@@" - docker exec "$master_name" kubectl logs -n "${ns}" -c "${container}" "${pod}" - done - echo "@@@ kubectl-all.txt @@@" - docker exec "$master_name" kubectl get all --all-namespaces -o wide - echo "@@@ describe-all.txt @@@" - docker exec "$master_name" kubectl describe all --all-namespaces - echo "@@@ nodes.txt @@@" - docker exec "$master_name" kubectl get nodes -o wide -} - -function dind::dump64 { - echo "%%% start-base64 %%%" - dind::dump | docker exec -i "$(dind::master-name)" /bin/sh -c "lzma | base64 -w 100" - echo "%%% end-base64 %%%" -} - -function dind::split-dump { - mkdir -p cluster-dump - cd cluster-dump - awk '!/^@@@ .* @@@$/{print >out}; /^@@@ .* @@@$/{out=$2}' out=/dev/null - ls -l -} - -function dind::split-dump64 { - decode_opt=-d - if base64 --help | grep -q '^ *-D'; then - # Mac OS X - decode_opt=-D - fi - sed -n '/^%%% start-base64 %%%$/,/^%%% end-base64 %%%$/p' | - sed '1d;$d' | - base64 "${decode_opt}" | - lzma -dc | - dind::split-dump -} - -function dind::proxy { - local container_id="$1" - if [[ ${DIND_CA_CERT_URL} ]] ; then - dind::step "+ Adding certificate on ${container_id}" - docker exec ${container_id} /bin/sh -c "cd /usr/local/share/ca-certificates; curl -sSO ${DIND_CA_CERT_URL}" - docker exec ${container_id} update-ca-certificates - fi - if [[ "${DIND_PROPAGATE_HTTP_PROXY}" || "${DIND_HTTP_PROXY}" || "${DIND_HTTPS_PROXY}" || "${DIND_NO_PROXY}" ]]; then - dind::step "+ Setting *_PROXY for docker service on ${container_id}" - local proxy_env="[Service]"$'\n'"Environment=" - if [[ "${DIND_PROPAGATE_HTTP_PROXY}" ]]; then - # take *_PROXY values from container environment - proxy_env+=$(docker exec ${container_id} env | grep -i _proxy | awk '{ print "\""$0"\""}' | xargs -d'\n') - else - if [[ "${DIND_HTTP_PROXY}" ]] ; then proxy_env+="\"HTTP_PROXY=${DIND_HTTP_PROXY}\" "; fi - if [[ "${DIND_HTTPS_PROXY}" ]] ; then proxy_env+="\"HTTPS_PROXY=${DIND_HTTPS_PROXY}\" "; fi - if [[ "${DIND_NO_PROXY}" ]] ; then proxy_env+="\"NO_PROXY=${DIND_NO_PROXY}\" "; fi - fi - docker exec -i ${container_id} /bin/sh -c "cat > /etc/systemd/system/docker.service.d/30-proxy.conf" <<< "${proxy_env}" - docker exec ${container_id} systemctl daemon-reload - docker exec ${container_id} systemctl restart docker - fi -} - -function dind::custom-docker-opts { - local container_id="$1" - local -a jq=() - local got_changes="" - if [[ ! -f ${DIND_DAEMON_JSON_FILE} ]] ; then - jq[0]="{}" - else - jq+=("$(cat ${DIND_DAEMON_JSON_FILE})") - if [[ ${DIND_DAEMON_JSON_FILE} != "/etc/docker/daemon.json" ]]; then - got_changes=1 - fi - fi - if [[ ${DIND_REGISTRY_MIRROR} ]] ; then - dind::step "+ Setting up registry mirror on ${container_id}" - jq+=("{\"registry-mirrors\": [\"${DIND_REGISTRY_MIRROR}\"]}") - got_changes=1 - fi - if [[ ${DIND_INSECURE_REGISTRIES} ]] ; then - dind::step "+ Setting up insecure-registries on ${container_id}" - jq+=("{\"insecure-registries\": ${DIND_INSECURE_REGISTRIES}}") - got_changes=1 - fi - if [[ ${got_changes} ]] ; then - local json=$(IFS="+"; echo "${jq[*]}") - docker exec -i ${container_id} /bin/sh -c "mkdir -p /etc/docker && jq -n '${json}' > /etc/docker/daemon.json" - docker exec ${container_id} systemctl daemon-reload - docker exec ${container_id} systemctl restart docker - fi -} - -COMMAND="${1:-}" - -case ${COMMAND} in - up) - if [[ ! ( ${DIND_IMAGE} =~ local ) && ! ${DIND_SKIP_PULL:-} ]]; then - dind::step "Making sure DIND image is up to date" - docker pull "${DIND_IMAGE}" >&2 - fi - - dind::prepare-sys-mounts - dind::ensure-kubectl - if [[ ${SKIP_SNAPSHOT} ]]; then - force_make_binaries=y dind::up - elif ! dind::check-for-snapshot; then - force_make_binaries=y dind::up - dind::snapshot - else - dind::restore - fi - ;; - reup) - dind::prepare-sys-mounts - dind::ensure-kubectl - if [[ ${SKIP_SNAPSHOT} ]]; then - force_make_binaries=y dind::up - elif ! dind::check-for-snapshot; then - force_make_binaries=y dind::up - dind::snapshot - else - force_make_binaries=y - restore_cmd=update_and_restore - dind::restore - fi - ;; - down) - dind::down - ;; - init) - shift - dind::prepare-sys-mounts - dind::ensure-kubectl - dind::init "$@" - ;; - join) - shift - dind::prepare-sys-mounts - dind::ensure-kubectl - dind::join "$(dind::create-node-container)" "$@" - ;; - # bare) - # shift - # dind::bare "$@" - # ;; - snapshot) - shift - dind::snapshot - ;; - restore) - shift - dind::restore - ;; - clean) - dind::clean - ;; - copy-image) - dind::copy-image "$@" - ;; - e2e) - shift - dind::run-e2e "$@" - ;; - e2e-serial) - shift - dind::run-e2e-serial "$@" - ;; - dump) - dind::dump - ;; - dump64) - dind::dump64 - ;; - split-dump) - dind::split-dump - ;; - split-dump64) - dind::split-dump64 - ;; - apiserver-port) - dind::apiserver-port - ;; - *) - echo "usage:" >&2 - echo " $0 up" >&2 - echo " $0 reup" >&2 - echo " $0 down" >&2 - echo " $0 init kubeadm-args..." >&2 - echo " $0 join kubeadm-args..." >&2 - # echo " $0 bare container_name [docker_options...]" - echo " $0 clean" - echo " $0 copy-image [image_name]" >&2 - echo " $0 e2e [test-name-substring]" >&2 - echo " $0 e2e-serial [test-name-substring]" >&2 - echo " $0 dump" >&2 - echo " $0 dump64" >&2 - echo " $0 split-dump" >&2 - echo " $0 split-dump64" >&2 - exit 1 - ;; -esac \ No newline at end of file diff --git a/ci/get-master-kubeconfig.sh b/ci/get-master-kubeconfig.sh deleted file mode 100755 index 4f769a5ab7f..00000000000 --- a/ci/get-master-kubeconfig.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -ROOT_DIR=$1 -SSH_USER="ClusterAPI" -# line in the cluster config file with the resource group name -RG_LINE=16 - -echo "Adding keys to agent" -eval `ssh-agent -s` -ssh-add $ROOT_DIR/cmd/clusterctl/examples/azure/out/sshkey - -rgName=$(cut -d ":" -f 2 <<<$(head -$RG_LINE $ROOT_DIR/cmd/clusterctl/examples/azure/out/cluster.yaml | tail -1) | tr -d '"') - -# get the master node ip address -echo "Looking for master VM in resource group: $rgName" -ip=$(az vm list-ip-addresses -g $rgName --query '[*].virtualMachine.network.publicIpAddresses[?starts_with(name, `ClusterAPIIP-azure-master`) == `true`].ipAddress | [0][0]' | tr -d '"') -echo "Getting kube config from vm with ip: $ip" -scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ClusterAPI@$ip:/home/ClusterAPI/.kube/config $ROOT_DIR/kubeconfig diff --git a/cmd/clusterctl/examples/azure/generate-yaml.sh b/cmd/clusterctl/examples/azure/generate-yaml.sh index bb9086520b0..692ab7eae29 100755 --- a/cmd/clusterctl/examples/azure/generate-yaml.sh +++ b/cmd/clusterctl/examples/azure/generate-yaml.sh @@ -1,3 +1,17 @@ +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + #!/bin/bash set -o errexit diff --git a/cmd/clusterctl/main.go b/cmd/clusterctl/main.go index 658eb792ca5..24e085fd546 100644 --- a/cmd/clusterctl/main.go +++ b/cmd/clusterctl/main.go @@ -1,3 +1,19 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package main import ( diff --git a/cmd/manager/main.go b/cmd/manager/main.go index a634a304c8c..2405289518e 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -1,4 +1,5 @@ /* +Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/hack/boilerplate/BUILD b/hack/boilerplate/BUILD new file mode 100644 index 00000000000..20e91a3060d --- /dev/null +++ b/hack/boilerplate/BUILD @@ -0,0 +1,31 @@ +package(default_visibility = ["//visibility:public"]) + +exports_files(glob(["*.txt"])) + +py_test( + name = "boilerplate_test", + srcs = [ + "boilerplate.py", + "boilerplate_test.py", + ], + data = glob([ + "*.txt", + "test/*", + ]), +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [ + ":package-srcs", + "//hack/boilerplate/test:all-srcs", + ], + tags = ["automanaged"], +) diff --git a/hack/boilerplate/boilerplate.Dockerfile.txt b/hack/boilerplate/boilerplate.Dockerfile.txt new file mode 100644 index 00000000000..384f325abf3 --- /dev/null +++ b/hack/boilerplate/boilerplate.Dockerfile.txt @@ -0,0 +1,14 @@ +# Copyright YEAR The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/hack/boilerplate/boilerplate.Makefile.txt b/hack/boilerplate/boilerplate.Makefile.txt new file mode 100644 index 00000000000..384f325abf3 --- /dev/null +++ b/hack/boilerplate/boilerplate.Makefile.txt @@ -0,0 +1,14 @@ +# Copyright YEAR The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/hack/boilerplate/boilerplate.bzl.txt b/hack/boilerplate/boilerplate.bzl.txt new file mode 100644 index 00000000000..384f325abf3 --- /dev/null +++ b/hack/boilerplate/boilerplate.bzl.txt @@ -0,0 +1,14 @@ +# Copyright YEAR The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/hack/boilerplate/boilerplate.generatebzl.txt b/hack/boilerplate/boilerplate.generatebzl.txt new file mode 100644 index 00000000000..069e282bc85 --- /dev/null +++ b/hack/boilerplate/boilerplate.generatebzl.txt @@ -0,0 +1,14 @@ +# Copyright The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/hack/boilerplate/boilerplate.generatego.txt b/hack/boilerplate/boilerplate.generatego.txt new file mode 100644 index 00000000000..b7c650da470 --- /dev/null +++ b/hack/boilerplate/boilerplate.generatego.txt @@ -0,0 +1,16 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + diff --git a/hack/boilerplate.go.txt b/hack/boilerplate/boilerplate.go.txt similarity index 92% rename from hack/boilerplate.go.txt rename to hack/boilerplate/boilerplate.go.txt index b92001fb4ed..59e740c1ee4 100644 --- a/hack/boilerplate.go.txt +++ b/hack/boilerplate/boilerplate.go.txt @@ -1,4 +1,5 @@ /* +Copyright YEAR The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -11,4 +12,5 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -*/ \ No newline at end of file +*/ + diff --git a/hack/boilerplate/boilerplate.py b/hack/boilerplate/boilerplate.py new file mode 100755 index 00000000000..fc1950fb27b --- /dev/null +++ b/hack/boilerplate/boilerplate.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import argparse +import datetime +import difflib +import glob +import os +import re +import sys + +parser = argparse.ArgumentParser() +parser.add_argument( + "filenames", + help="list of files to check, all files if unspecified", + nargs='*') + +rootdir = os.path.dirname(__file__) + "/../../" +rootdir = os.path.abspath(rootdir) +parser.add_argument( + "--rootdir", default=rootdir, help="root directory to examine") + +default_boilerplate_dir = os.path.join(rootdir, "hack/boilerplate") +parser.add_argument( + "--boilerplate-dir", default=default_boilerplate_dir) + +parser.add_argument( + "-v", "--verbose", + help="give verbose output regarding why a file does not pass", + action="store_true") + +args = parser.parse_args() + +verbose_out = sys.stderr if args.verbose else open("/dev/null", "w") + +def get_refs(): + refs = {} + + for path in glob.glob(os.path.join(args.boilerplate_dir, "boilerplate.*.txt")): + extension = os.path.basename(path).split(".")[1] + + ref_file = open(path, 'r') + ref = ref_file.read().splitlines() + ref_file.close() + refs[extension] = ref + + return refs + +def is_generated_file(filename, data, regexs): + for d in skipped_ungenerated_files: + if d in filename: + return False + + p = regexs["generated"] + return p.search(data) + +def file_passes(filename, refs, regexs): + try: + f = open(filename, 'r') + except Exception as exc: + print("Unable to open %s: %s" % (filename, exc), file=verbose_out) + return False + + data = f.read() + f.close() + + # determine if the file is automatically generated + generated = is_generated_file(filename, data, regexs) + + basename = os.path.basename(filename) + extension = file_extension(filename) + if generated: + if extension == "go": + extension = "generatego" + elif extension == "bzl": + extension = "generatebzl" + + if extension != "": + ref = refs[extension] + else: + ref = refs[basename] + + # remove extra content from the top of files + if extension == "go" or extension == "generatego": + p = regexs["go_build_constraints"] + (data, found) = p.subn("", data, 1) + elif extension == "sh": + p = regexs["shebang"] + (data, found) = p.subn("", data, 1) + + data = data.splitlines() + + # if our test file is smaller than the reference it surely fails! + if len(ref) > len(data): + print('File %s smaller than reference (%d < %d)' % + (filename, len(data), len(ref)), + file=verbose_out) + return False + + # trim our file to the same number of lines as the reference file + data = data[:len(ref)] + + p = regexs["year"] + for d in data: + if p.search(d): + if generated: + print('File %s has the YEAR field, but it should not be in generated file' % filename, file=verbose_out) + else: + print('File %s has the YEAR field, but missing the year of date' % filename, file=verbose_out) + return False + + if not generated: + # Replace all occurrences of the regex "2014|2015|2016|2017|2018" with "YEAR" + p = regexs["date"] + for i, d in enumerate(data): + (data[i], found) = p.subn('YEAR', d) + if found != 0: + break + + # if we don't match the reference at this point, fail + if ref != data: + print("Header in %s does not match reference, diff:" % filename, file=verbose_out) + if args.verbose: + print(file=verbose_out) + for line in difflib.unified_diff(ref, data, 'reference', filename, lineterm=''): + print(line, file=verbose_out) + print(file=verbose_out) + return False + + return True + +def file_extension(filename): + return os.path.splitext(filename)[1].split(".")[-1].lower() + +skipped_dirs = ['bin', '.git', "vendor", "hack/boilerplate/test"] + +# list all the files contain 'DO NOT EDIT', but are not generated +skipped_ungenerated_files = ['hack/boilerplate/boilerplate.py'] + +def normalize_files(files): + newfiles = [] + for pathname in files: + if any(x in pathname for x in skipped_dirs): + continue + newfiles.append(pathname) + for i, pathname in enumerate(newfiles): + if not os.path.isabs(pathname): + newfiles[i] = os.path.join(args.rootdir, pathname) + return newfiles + +def get_files(extensions): + files = [] + if len(args.filenames) > 0: + files = args.filenames + else: + for root, dirs, walkfiles in os.walk(args.rootdir): + # don't visit certain dirs. This is just a performance improvement + # as we would prune these later in normalize_files(). But doing it + # cuts down the amount of filesystem walking we do and cuts down + # the size of the file list + for d in skipped_dirs: + if d in dirs: + dirs.remove(d) + + for name in walkfiles: + pathname = os.path.join(root, name) + files.append(pathname) + + files = normalize_files(files) + outfiles = [] + for pathname in files: + basename = os.path.basename(pathname) + extension = file_extension(pathname) + if extension in extensions or basename in extensions: + outfiles.append(pathname) + return outfiles + +def get_dates(): + years = datetime.datetime.now().year + return '(%s)' % '|'.join((str(year) for year in range(2014, years+1))) + +def get_regexs(): + regexs = {} + # Search for "YEAR" which exists in the boilerplate, but shouldn't in the real thing + regexs["year"] = re.compile( 'YEAR' ) + # get_dates return 2014, 2015, 2016, 2017, or 2018 until the current year as a regex like: "(2014|2015|2016|2017|2018)"; + # company holder names can be anything + regexs["date"] = re.compile(get_dates()) + # strip // +build \n\n build constraints + regexs["go_build_constraints"] = re.compile(r"^(// \+build.*\n)+\n", re.MULTILINE) + # strip #!.* from shell scripts + regexs["shebang"] = re.compile(r"^(#!.*\n)\n*", re.MULTILINE) + # Search for generated files + regexs["generated"] = re.compile( 'DO NOT EDIT' ) + return regexs + +def main(): + regexs = get_regexs() + refs = get_refs() + filenames = get_files(refs.keys()) + + for filename in filenames: + if not file_passes(filename, refs, regexs): + print(filename, file=sys.stdout) + + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/hack/boilerplate/boilerplate.py.txt b/hack/boilerplate/boilerplate.py.txt new file mode 100644 index 00000000000..a2e72e5988e --- /dev/null +++ b/hack/boilerplate/boilerplate.py.txt @@ -0,0 +1,16 @@ +#!/usr/bin/env python + +# Copyright YEAR The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/hack/boilerplate/boilerplate.sh.txt b/hack/boilerplate/boilerplate.sh.txt new file mode 100644 index 00000000000..384f325abf3 --- /dev/null +++ b/hack/boilerplate/boilerplate.sh.txt @@ -0,0 +1,14 @@ +# Copyright YEAR The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/hack/boilerplate/boilerplate_test.py b/hack/boilerplate/boilerplate_test.py new file mode 100644 index 00000000000..b8d5b8e9e0f --- /dev/null +++ b/hack/boilerplate/boilerplate_test.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python + +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import boilerplate +import unittest +import StringIO +import os +import sys + +class TestBoilerplate(unittest.TestCase): + """ + Note: run this test from the hack/boilerplate directory. + + $ python -m unittest boilerplate_test + """ + + def test_boilerplate(self): + os.chdir("test/") + + class Args(object): + def __init__(self): + self.filenames = [] + self.rootdir = "." + self.boilerplate_dir = "../" + self.verbose = True + + # capture stdout + old_stdout = sys.stdout + sys.stdout = StringIO.StringIO() + + boilerplate.args = Args() + ret = boilerplate.main() + + output = sorted(sys.stdout.getvalue().split()) + + sys.stdout = old_stdout + + self.assertEquals( + output, ['././fail.go', '././fail.py']) diff --git a/hack/boilerplate/test/BUILD b/hack/boilerplate/test/BUILD new file mode 100644 index 00000000000..d6975423bb4 --- /dev/null +++ b/hack/boilerplate/test/BUILD @@ -0,0 +1,28 @@ +package(default_visibility = ["//visibility:public"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", +) + +go_library( + name = "go_default_library", + srcs = [ + "fail.go", + "pass.go", + ], + importpath = "k8s.io/kubernetes/hack/boilerplate/test", +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/hack/boilerplate/test/fail.go b/hack/boilerplate/test/fail.go new file mode 100644 index 00000000000..16159c5ac0d --- /dev/null +++ b/hack/boilerplate/test/fail.go @@ -0,0 +1,19 @@ +/* +Copyright 2014 The Kubernetes Authors. + +fail + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test diff --git a/hack/boilerplate/test/fail.py b/hack/boilerplate/test/fail.py new file mode 100644 index 00000000000..cbdd06ff8a1 --- /dev/null +++ b/hack/boilerplate/test/fail.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python + +# Copyright 2015 The Kubernetes Authors. +# +# failed +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/hack/boilerplate/test/pass.go b/hack/boilerplate/test/pass.go new file mode 100644 index 00000000000..7508448aae6 --- /dev/null +++ b/hack/boilerplate/test/pass.go @@ -0,0 +1,17 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test diff --git a/hack/boilerplate/test/pass.py b/hack/boilerplate/test/pass.py new file mode 100644 index 00000000000..5b7ce29a25c --- /dev/null +++ b/hack/boilerplate/test/pass.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python + +# Copyright 2015 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +True diff --git a/hack/verify-boilerplate.sh b/hack/verify-boilerplate.sh new file mode 100755 index 00000000000..cb45b73dde8 --- /dev/null +++ b/hack/verify-boilerplate.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Copyright 2014 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. + +boilerDir="${KUBE_ROOT}/hack/boilerplate" +boiler="${boilerDir}/boilerplate.py" + +files_need_boilerplate=($(${boiler} "$@")) + +# Run boilerplate check +if [[ ${#files_need_boilerplate[@]} -gt 0 ]]; then + for file in "${files_need_boilerplate[@]}"; do + echo "Boilerplate header is wrong for: ${file}" >&2 + done + + exit 1 +fi diff --git a/pkg/apis/addtoscheme_azureprovider_v1alpha1.go b/pkg/apis/addtoscheme_azureprovider_v1alpha1.go index 357b0059363..07f25e8c003 100644 --- a/pkg/apis/addtoscheme_azureprovider_v1alpha1.go +++ b/pkg/apis/addtoscheme_azureprovider_v1alpha1.go @@ -1,4 +1,5 @@ /* +Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/apis/apis.go b/pkg/apis/apis.go index 15ccbf9c8a5..b8b48e9c8b0 100644 --- a/pkg/apis/apis.go +++ b/pkg/apis/apis.go @@ -1,4 +1,5 @@ /* +Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,7 +15,7 @@ limitations under the License. */ // Generate deepcopy for apis -//go:generate go run ../../vendor/k8s.io/code-generator/cmd/deepcopy-gen/main.go -O zz_generated.deepcopy -i ./... -h ../../hack/boilerplate.go.txt +//go:generate go run ../../vendor/k8s.io/code-generator/cmd/deepcopy-gen/main.go -O zz_generated.deepcopy -i ./... -h ../../hack/boilerplate/boilerplate.generatego.txt // Package apis contains Kubernetes API groups. package apis diff --git a/pkg/apis/azureprovider/group.go b/pkg/apis/azureprovider/group.go index b977a261f93..b4ae12c862d 100644 --- a/pkg/apis/azureprovider/group.go +++ b/pkg/apis/azureprovider/group.go @@ -1,4 +1,5 @@ /* +Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -13,5 +14,4 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package azureprovider contains azureprovider API versions package azureprovider diff --git a/pkg/apis/azureprovider/v1alpha1/azureclusterproviderconfig_types.go b/pkg/apis/azureprovider/v1alpha1/azureclusterproviderconfig_types.go index 41ba2548b9c..a0a011069d5 100644 --- a/pkg/apis/azureprovider/v1alpha1/azureclusterproviderconfig_types.go +++ b/pkg/apis/azureprovider/v1alpha1/azureclusterproviderconfig_types.go @@ -1,4 +1,5 @@ /* +Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/apis/azureprovider/v1alpha1/azuremachineproviderconfig_types.go b/pkg/apis/azureprovider/v1alpha1/azuremachineproviderconfig_types.go index fa0de063adf..d06b25660ce 100644 --- a/pkg/apis/azureprovider/v1alpha1/azuremachineproviderconfig_types.go +++ b/pkg/apis/azureprovider/v1alpha1/azuremachineproviderconfig_types.go @@ -1,4 +1,5 @@ /* +Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/apis/azureprovider/v1alpha1/doc.go b/pkg/apis/azureprovider/v1alpha1/doc.go index ecae29ed277..53730f639ed 100644 --- a/pkg/apis/azureprovider/v1alpha1/doc.go +++ b/pkg/apis/azureprovider/v1alpha1/doc.go @@ -1,4 +1,5 @@ /* +Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/apis/azureprovider/v1alpha1/register.go b/pkg/apis/azureprovider/v1alpha1/register.go index 1d23207118e..501920448de 100644 --- a/pkg/apis/azureprovider/v1alpha1/register.go +++ b/pkg/apis/azureprovider/v1alpha1/register.go @@ -1,4 +1,5 @@ /* +Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/apis/azureprovider/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/azureprovider/v1alpha1/zz_generated.deepcopy.go index 0b7fad0e642..5b6b9e219b8 100644 --- a/pkg/apis/azureprovider/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/azureprovider/v1alpha1/zz_generated.deepcopy.go @@ -1,6 +1,7 @@ // +build !ignore_autogenerated /* +Copyright The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,6 +15,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + // Code generated by main. DO NOT EDIT. package v1alpha1 diff --git a/pkg/cloud/azure/actuators/cluster/actuator.go b/pkg/cloud/azure/actuators/cluster/actuator.go index b45fef7439e..5a23fa48a66 100644 --- a/pkg/cloud/azure/actuators/cluster/actuator.go +++ b/pkg/cloud/azure/actuators/cluster/actuator.go @@ -1,9 +1,12 @@ /* Copyright 2018 The Kubernetes Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/pkg/cloud/azure/actuators/cluster/actuator_test.go b/pkg/cloud/azure/actuators/cluster/actuator_test.go index 9f0d466e333..7f95b40b464 100644 --- a/pkg/cloud/azure/actuators/cluster/actuator_test.go +++ b/pkg/cloud/azure/actuators/cluster/actuator_test.go @@ -1,15 +1,19 @@ /* Copyright 2018 The Kubernetes Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + package cluster import ( diff --git a/pkg/cloud/azure/actuators/machine/actuator.go b/pkg/cloud/azure/actuators/machine/actuator.go index d37d2e2b7e9..6ef85fe028f 100644 --- a/pkg/cloud/azure/actuators/machine/actuator.go +++ b/pkg/cloud/azure/actuators/machine/actuator.go @@ -1,9 +1,12 @@ /* Copyright 2018 The Kubernetes Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/pkg/cloud/azure/actuators/machine/actuator_test.go b/pkg/cloud/azure/actuators/machine/actuator_test.go index 31bf80d8423..a0040e8cdb9 100644 --- a/pkg/cloud/azure/actuators/machine/actuator_test.go +++ b/pkg/cloud/azure/actuators/machine/actuator_test.go @@ -1,15 +1,19 @@ /* Copyright 2018 The Kubernetes Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + package machine import ( diff --git a/pkg/cloud/azure/actuators/machine/status.go b/pkg/cloud/azure/actuators/machine/status.go index aa321069e86..8e1d8a4a246 100644 --- a/pkg/cloud/azure/actuators/machine/status.go +++ b/pkg/cloud/azure/actuators/machine/status.go @@ -1,9 +1,12 @@ /* Copyright 2018 The Kubernetes Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/pkg/cloud/azure/services/compute/disks.go b/pkg/cloud/azure/services/compute/disks.go index e264e818723..2d26ff82e76 100644 --- a/pkg/cloud/azure/services/compute/disks.go +++ b/pkg/cloud/azure/services/compute/disks.go @@ -1,9 +1,12 @@ /* Copyright 2018 The Kubernetes Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/pkg/cloud/azure/services/compute/service.go b/pkg/cloud/azure/services/compute/service.go index 1254564d0fe..29c44f3124b 100644 --- a/pkg/cloud/azure/services/compute/service.go +++ b/pkg/cloud/azure/services/compute/service.go @@ -1,9 +1,12 @@ /* Copyright 2018 The Kubernetes Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/pkg/cloud/azure/services/compute/virtualmachines.go b/pkg/cloud/azure/services/compute/virtualmachines.go index 0c6cf491791..a78ebdea6c8 100644 --- a/pkg/cloud/azure/services/compute/virtualmachines.go +++ b/pkg/cloud/azure/services/compute/virtualmachines.go @@ -1,9 +1,12 @@ /* Copyright 2018 The Kubernetes Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/pkg/cloud/azure/services/interfaces.go b/pkg/cloud/azure/services/interfaces.go index aa1000bbace..976f7283618 100644 --- a/pkg/cloud/azure/services/interfaces.go +++ b/pkg/cloud/azure/services/interfaces.go @@ -1,9 +1,12 @@ /* Copyright 2018 The Kubernetes Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/pkg/cloud/azure/services/mock_interfaces.go b/pkg/cloud/azure/services/mock_interfaces.go index f0bf468e68b..345ce88ef17 100644 --- a/pkg/cloud/azure/services/mock_interfaces.go +++ b/pkg/cloud/azure/services/mock_interfaces.go @@ -1,9 +1,12 @@ /* Copyright 2018 The Kubernetes Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/pkg/cloud/azure/services/mock_interfaces_helper.go b/pkg/cloud/azure/services/mock_interfaces_helper.go index 61bcae87edc..af5ee13a2b3 100644 --- a/pkg/cloud/azure/services/mock_interfaces_helper.go +++ b/pkg/cloud/azure/services/mock_interfaces_helper.go @@ -1,9 +1,12 @@ /* Copyright 2018 The Kubernetes Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/pkg/cloud/azure/services/network/interfaces.go b/pkg/cloud/azure/services/network/interfaces.go index 4ed953afdb1..5806d33abc0 100644 --- a/pkg/cloud/azure/services/network/interfaces.go +++ b/pkg/cloud/azure/services/network/interfaces.go @@ -1,9 +1,12 @@ /* Copyright 2018 The Kubernetes Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/pkg/cloud/azure/services/network/publicipaddress.go b/pkg/cloud/azure/services/network/publicipaddress.go index b47ad571060..90673bfff39 100644 --- a/pkg/cloud/azure/services/network/publicipaddress.go +++ b/pkg/cloud/azure/services/network/publicipaddress.go @@ -1,9 +1,12 @@ /* Copyright 2018 The Kubernetes Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/pkg/cloud/azure/services/network/securitygroups.go b/pkg/cloud/azure/services/network/securitygroups.go index 42fd6683420..512207cc06a 100644 --- a/pkg/cloud/azure/services/network/securitygroups.go +++ b/pkg/cloud/azure/services/network/securitygroups.go @@ -1,9 +1,12 @@ /* Copyright 2018 The Kubernetes Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/pkg/cloud/azure/services/network/service.go b/pkg/cloud/azure/services/network/service.go index 35b74222963..9cac62c68b5 100644 --- a/pkg/cloud/azure/services/network/service.go +++ b/pkg/cloud/azure/services/network/service.go @@ -1,9 +1,12 @@ /* Copyright 2018 The Kubernetes Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/pkg/cloud/azure/services/network/virtualnetworks.go b/pkg/cloud/azure/services/network/virtualnetworks.go index 3f5da7aeda7..741f46dbab1 100644 --- a/pkg/cloud/azure/services/network/virtualnetworks.go +++ b/pkg/cloud/azure/services/network/virtualnetworks.go @@ -1,9 +1,12 @@ /* Copyright 2018 The Kubernetes Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/pkg/cloud/azure/services/resources/deployments.go b/pkg/cloud/azure/services/resources/deployments.go index f930c309061..e9fd0cdd2df 100644 --- a/pkg/cloud/azure/services/resources/deployments.go +++ b/pkg/cloud/azure/services/resources/deployments.go @@ -1,9 +1,12 @@ /* Copyright 2018 The Kubernetes Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/pkg/cloud/azure/services/resources/groups.go b/pkg/cloud/azure/services/resources/groups.go index 62fac4f1a5d..0bb034f4163 100644 --- a/pkg/cloud/azure/services/resources/groups.go +++ b/pkg/cloud/azure/services/resources/groups.go @@ -1,9 +1,12 @@ /* Copyright 2018 The Kubernetes Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/pkg/cloud/azure/services/resources/service.go b/pkg/cloud/azure/services/resources/service.go index 4bd69a1a456..2362ccf441f 100644 --- a/pkg/cloud/azure/services/resources/service.go +++ b/pkg/cloud/azure/services/resources/service.go @@ -1,9 +1,12 @@ /* Copyright 2018 The Kubernetes Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/test/e2e/azure.go b/test/e2e/azure.go index c554925e102..75743727614 100644 --- a/test/e2e/azure.go +++ b/test/e2e/azure.go @@ -1,9 +1,12 @@ /* Copyright 2018 The Kubernetes Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 82f3321280c..6c58c9ac6db 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -1,9 +1,12 @@ /* Copyright 2018 The Kubernetes Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/test/e2e/kube.go b/test/e2e/kube.go index 5d4088562be..cc461f64b84 100644 --- a/test/e2e/kube.go +++ b/test/e2e/kube.go @@ -1,9 +1,12 @@ /* Copyright 2018 The Kubernetes Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.