From 6327a368af7e2fe383d2ae8c87f1d5a73dd0e92c Mon Sep 17 00:00:00 2001 From: Ned Petrov Date: Fri, 3 Jul 2026 09:05:42 +0300 Subject: [PATCH] markdown int --- content/about.md | 3 - content/addons-common.md | 4 + content/agent-cpi-interactions.md | 54 +-- content/alicloud-cpi-errors.md | 3 +- content/alicloud-cpi.md | 167 +++++---- content/aws-cpi-errors.md | 13 +- content/aws-cpi.md | 167 ++++----- content/aws-iam-instance-profiles.md | 6 +- content/aws-iam-users.md | 2 + content/aws-instance-storage.md | 2 + content/aws.md | 32 +- content/azs.md | 105 +++--- content/azure-compute-gallery.md | 10 +- content/azure-cpi-errors.md | 65 ++-- content/azure-cpi.md | 321 +++++++++-------- content/azure-managed-identity.md | 2 + content/azure-resources.md | 17 +- content/azure.md | 29 +- content/basic-workflow.md | 2 + content/blobstore-agent-password-rotation.md | 30 +- content/blobstore-ca-rotation.md | 25 +- content/bosh-cli.md | 14 +- content/bosh-components.md | 16 +- content/bosh-lite.md | 10 +- content/build-cpi.md | 25 +- content/build-stemcell.md | 40 ++- content/cck.md | 36 +- content/changing-deployment-vm-strategy.md | 12 +- content/cli-env-deps.md | 4 + content/cli-envs.md | 6 + content/cli-global-flags.md | 12 +- content/cli-help.md | 4 - content/cli-int.md | 7 +- content/cli-ops-files.md | 21 +- content/cli-tunnel.md | 6 +- content/cli-v2-diff.md | 43 +-- content/cli-v2-install.md | 10 +- content/cli-v2.md | 35 +- content/cloud-config.md | 48 ++- content/community.md | 7 +- content/compiled-releases.md | 4 + content/configs.md | 12 +- content/cpi-api-rpc.md | 84 +++-- content/cpi-api-v1-method/attach-disk.md | 53 ++- .../cpi-api-v1-method/configure-networks.md | 7 +- content/cpi-api-v1-method/create-vm.md | 31 +- content/cpi-api-v1-method/current-vm-id.md | 4 +- content/cpi-api-v1-method/delete-vm.md | 8 +- content/cpi-api-v1-method/detach-disk.md | 15 +- content/cpi-api-v1-method/info.md | 6 +- content/cpi-api-v1.md | 3 +- content/cpi-api-v2-method/attach-disk.md | 32 +- .../calculate-vm-cloud-properties.md | 17 +- content/cpi-api-v2-method/create-disk.md | 19 +- content/cpi-api-v2-method/create-network.md | 13 +- content/cpi-api-v2-method/create-stemcell.md | 45 ++- content/cpi-api-v2-method/create-vm.md | 38 +- content/cpi-api-v2-method/delete-disk.md | 13 +- content/cpi-api-v2-method/delete-network.md | 10 +- content/cpi-api-v2-method/delete-snapshot.md | 7 +- content/cpi-api-v2-method/delete-stemcell.md | 11 +- content/cpi-api-v2-method/delete-vm.md | 10 +- content/cpi-api-v2-method/detach-disk.md | 19 +- content/cpi-api-v2-method/get-disks.md | 11 +- content/cpi-api-v2-method/gopher.jpg | Bin 1754 -> 0 bytes content/cpi-api-v2-method/has-disk.md | 13 +- content/cpi-api-v2-method/has-vm.md | 13 +- content/cpi-api-v2-method/info.md | 9 +- content/cpi-api-v2-method/reboot-vm.md | 8 +- content/cpi-api-v2-method/resize-disk.md | 16 +- .../cpi-api-v2-method/set-disk-metadata.md | 14 +- content/cpi-api-v2-method/set-vm-metadata.md | 13 +- content/cpi-api-v2-method/snapshot-disk.md | 11 +- content/cpi-api-v2-method/update-disk.md | 16 +- content/cpi-api-v2-migration-guide.md | 126 +++---- content/cpi-api-v2.md | 52 +-- content/cpi-api-v3-method/create-stemcell.md | 44 ++- content/cpi-api-v3.md | 6 +- content/cpi-config.md | 15 +- content/create-release.md | 112 +++--- .../credhub-encryption-password-rotation.md | 18 +- content/creds-tmpfs.md | 10 +- content/deploy-config.md | 7 +- content/deploying-step-by-step.md | 3 + content/deploying.md | 6 +- content/deployment-basics.md | 4 +- content/deployment-convergence.md | 2 + content/deployment-manifest.md | 158 +++++---- content/deployment.md | 2 + content/design.md | 11 +- content/director-access-events.md | 7 +- content/director-api-v1.md | 324 ++++++++++-------- content/director-backup.md | 41 ++- content/director-blobstore-signed-urls.md | 56 ++- content/director-bosh-teams.md | 6 +- content/director-certs-openssl.md | 4 + content/director-certs.md | 4 +- content/director-configure-blobstore.md | 9 +- content/director-configure-db.md | 3 + content/director-tasks.md | 21 +- content/director-users-uaa-scopes.md | 14 +- content/director-users-uaa.md | 3 + content/director-users.md | 7 + content/disaster-recovery.md | 14 +- content/dns.md | 173 ++++++---- content/drain.md | 27 +- content/errands.md | 11 +- content/events.md | 6 +- content/flush-arp.md | 2 + content/google-cpi.md | 65 ++-- content/google-required-permissions.md | 14 +- content/google.md | 30 +- content/guide-ipv6-on-vsphere.md | 2 + content/guide-multi-cpi-aws.md | 20 +- content/hm-config.md | 10 + content/index.md | 2 - content/init-alicloud.md | 50 ++- content/init-aws.md | 62 ++-- content/init-azure.md | 3 + content/init-external-ip.md | 2 + content/init-google.md | 2 + content/init-openstack.md | 73 ++-- content/init-vsphere-rp.md | 2 + content/init-vsphere.md | 2 + content/init.md | 2 + content/instance-metadata.md | 12 +- content/jammy-migration.md | 24 +- content/job-lifecycle.md | 13 +- content/job-logs.md | 16 +- content/job-templates.md | 2 + content/job-tmpfs.md | 61 ---- content/jobs.md | 89 ++--- content/jumpbox.md | 2 + content/links-api.md | 55 +-- content/links-common-types.md | 24 +- content/links-manual.md | 2 + content/links-properties.md | 5 +- content/links.md | 97 +++--- content/locking-compiled-releases.md | 9 +- content/managed-networks.md | 10 +- content/managing-releases.md | 9 +- content/managing-stemcells.md | 6 + content/manifest-v2.md | 226 ++++++------ content/mbus-ssl-rotation.md | 14 +- content/migrated-from.md | 9 +- content/monitoring.md | 10 +- content/nats-ca-rotation.md | 27 +- content/networks.md | 115 ++++--- content/noble-migration.md | 14 +- content/ntp-config.md | 5 +- content/openstack-cpi-errors.md | 28 +- content/openstack-cpi.md | 42 ++- content/openstack-human-readable-vm-names.md | 2 + content/openstack-keystonev2.md | 2 + content/openstack-light-stemcells.md | 10 +- content/openstack-multiple-networks.md | 10 +- content/openstack-registry.md | 2 + content/openstack-self-signed-endpoints.md | 2 + content/openstack-vrrp.md | 36 +- content/openstack.md | 52 ++- content/package-vendoring.md | 15 +- content/packages.md | 19 +- content/persistent-disk-fs.md | 2 + content/persistent-disks.md | 40 ++- content/personas.md | 113 +++--- content/post-deploy.md | 6 + content/post-start.md | 5 + content/post-stop.md | 4 + content/pre-start.md | 5 + content/pre-stop.md | 24 +- content/problems.md | 2 + content/props-common.md | 21 +- content/quick-start.md | 15 +- content/rackhd-cpi.md | 2 + content/recent.md | 50 +-- content/recover.md | 12 +- content/release-blobs.md | 5 + content/release-blobstore.md | 22 +- content/release-urls.md | 4 + content/release.md | 2 + content/remove-dev-tools.md | 2 + content/repack-stemcell.md | 9 +- content/resurrector.md | 167 ++++----- content/runtime-config.md | 52 +-- content/s3-release-blobstore.md | 4 +- content/sample-manifest.md | 2 + content/scheduled-procs.md | 2 + content/scheduled-task-cleanup.md | 14 +- content/snapshots.md | 10 +- content/stemcell.md | 23 +- content/sysadmin-commands.md | 18 + content/terminology.md | 50 +++ content/tips.md | 34 +- content/troubleshooting.md | 5 +- content/trusted-certs.md | 3 + content/understanding-bosh.md | 30 +- content/update-cloud-config.md | 2 + content/uploading-releases.md | 10 +- content/uploading-stemcells.md | 6 +- content/variable-types.md | 50 +-- content/virtualbox-cpi.md | 19 +- content/vm-anti-affinity.md | 18 +- content/vm-config.md | 54 +-- content/vm-monit.md | 13 +- content/vm-struct.md | 2 + content/vsphere-cpi-errors.md | 28 +- content/vsphere-cpi.md | 208 +++++------ content/vsphere-esxi-host-failure.md | 34 +- content/vsphere-ha.md | 12 +- content/vsphere-human-readable-names.md | 44 +-- content/vsphere-migrate-datastores.md | 6 +- content/vsphere-network-partition.md | 29 +- content/vsphere-vmotion-support.md | 10 +- content/vsphere.md | 30 +- content/warden-cpi.md | 18 +- content/windows-sample-release.md | 7 + content/windows-stemcell-create.md | 123 ++++--- content/windows.md | 10 +- mkdocs.yml | 3 + 219 files changed, 3385 insertions(+), 2835 deletions(-) delete mode 100644 content/about.md delete mode 100644 content/cli-help.md delete mode 100644 content/cpi-api-v2-method/gopher.jpg delete mode 100644 content/job-tmpfs.md diff --git a/content/about.md b/content/about.md deleted file mode 100644 index a0cc1ab28..000000000 --- a/content/about.md +++ /dev/null @@ -1,3 +0,0 @@ -BOSH is a project that unifies release engineering, deployment, and lifecycle management of small and large-scale cloud software. BOSH can provision and deploy software over hundreds of VMs. It also performs monitoring, failure recovery, and software updates with zero-to-minimal downtime. - -While BOSH was developed to deploy Cloud Foundry PaaS, it can also be used to deploy almost any other software (Hadoop, for instance). BOSH is particularly well-suited for large distributed systems. In addition, BOSH supports multiple Infrastructure as a Service (IaaS) providers like VMware vSphere, Google Cloud Platform, Amazon Web Services EC2, OpenStack, and Alibaba Cloud. There is a Cloud Provider Interface (CPI) that enables users to extend BOSH to support additional IaaS providers such as Apache CloudStack and VirtualBox. diff --git a/content/addons-common.md b/content/addons-common.md index 29883e2ae..731eef037 100644 --- a/content/addons-common.md +++ b/content/addons-common.md @@ -1,3 +1,5 @@ +# Common Addons + (See [runtime config](runtime-config.md#addons) for an introduction to addons.) ## Syslog forwarding {: #syslog } @@ -30,6 +32,7 @@ addons: See [syslog_forwarder job](https://bosh.io/jobs/syslog_forwarder?source=github.com/cloudfoundry/syslog-release). --- + ## Custom SSH login banner {: #login-banner } !!! note @@ -59,6 +62,7 @@ addons: See [login_banner job](https://bosh.io/jobs/login_banner?source=github.com/cloudfoundry/os-conf-release). --- + ## Custom SSH users {: #misc-users } !!! warning diff --git a/content/agent-cpi-interactions.md b/content/agent-cpi-interactions.md index bea280f67..e5ee6eff4 100644 --- a/content/agent-cpi-interactions.md +++ b/content/agent-cpi-interactions.md @@ -1,10 +1,12 @@ +# Agent Interactions + Here is an overview of the interactions between the CPI and the Agent on CloudStack as an example: -* The CPI drives the IaaS and the Agent. -* The Agent is a versatile process which configures the OS to leverage IaaS-provisioned resources (network interfaces, disks, etc.), and perform other BOSH tasks (job compilation, job instantiation, etc.) -* The CPI asks the IaaS to instantiate VM template, VMs, volumes and possibly other constructs (floating IPs, security groups, connect LBs, etc.) -* The director-to-agent communication happens through NATS-based messaging. -* The initial configuration of the agent is done via the metadata server. +- The CPI drives the IaaS and the Agent. +- The Agent is a versatile process which configures the OS to leverage IaaS-provisioned resources (network interfaces, disks, etc.), and perform other BOSH tasks (job compilation, job instantiation, etc.) +- The CPI asks the IaaS to instantiate VM template, VMs, volumes and possibly other constructs (floating IPs, security groups, connect LBs, etc.) +- The director-to-agent communication happens through NATS-based messaging. +- The initial configuration of the agent is done via the metadata server. !!! note If either of CPI, Director or stemcell does not support [CPI API version 2](cpi-api-v2.md#reference-table-based-on-each-component-version), @@ -85,6 +87,7 @@ Render it online http://plantuml.com/plantuml/ or from a private plantuml instan --> --- + ## Agent settings {: #agent-settings} This section details the configuration and protocols supported by the Agent. @@ -92,6 +95,7 @@ This section details the configuration and protocols supported by the Agent. [VM Configuration Locations](vm-config.html#agent) provides a list of Agent configuration files and their roles. --- + ### `agent.json` file {: #agent-json } `/var/vcap/bosh/agent.json`: Start up settings for the Agent that describe how to find bootstrap settings, and disable certain Agent functionality. @@ -102,11 +106,11 @@ The Platform part of the file is documented into [LinuxOptions](https://pkg.go.d The Infrastructure part of the file is documented into [Options](https://pkg.go.dev/github.com/cloudfoundry/bosh-agent/v2/infrastructure#Options) and in particular: - * the sources of configuration with [SettingsOptions](https://pkg.go.dev/github.com/cloudfoundry/bosh-agent/v2/infrastructure#SettingsOptions): +- the sources of configuration with [SettingsOptions](https://pkg.go.dev/github.com/cloudfoundry/bosh-agent/v2/infrastructure#SettingsOptions): - * [CDROM](https://pkg.go.dev/github.com/cloudfoundry/bosh-agent/v2/infrastructure#CDROMSourceOptions) - * [ConfigDrive](https://pkg.go.dev/github.com/cloudfoundry/bosh-agent/v2/infrastructure#ConfigDriveSourceOptions) - * [HTTP](https://pkg.go.dev/github.com/cloudfoundry/bosh-agent/v2/infrastructure#HTTPSourceOptions) + - [CDROM](https://pkg.go.dev/github.com/cloudfoundry/bosh-agent/v2/infrastructure#CDROMSourceOptions) + - [ConfigDrive](https://pkg.go.dev/github.com/cloudfoundry/bosh-agent/v2/infrastructure#ConfigDriveSourceOptions) + - [HTTP](https://pkg.go.dev/github.com/cloudfoundry/bosh-agent/v2/infrastructure#HTTPSourceOptions) Sample `agent.json` which configures the agent to read from an HTTP metadata service at a custom URL: @@ -168,6 +172,7 @@ Sample `agent.json` which configures the agent to read from a config drive. See ``` --- + ### Agent settings format {: #agent-settings-format} The current state of the JSON settings format supported by the Agent is documented in [settings_test.go](https://github.com/cloudfoundry/bosh-agent/blob/master/settings/settings_test.go) as well as in [settings.go](https://github.com/cloudfoundry/bosh-agent/blob/master/settings/settings.go). @@ -261,7 +266,8 @@ Sample settings JSON: } ``` ----- +--- + ### settings.json file {: #settings-json } `/var/vcap/bosh/settings.json`: Local copy of the bootstrap settings used by the Agent to configure network and system properties for the VM. They are refreshed every time Agent is restarted. @@ -269,26 +275,29 @@ Sample settings JSON: The `settings.json` format is the same as the settings format initially transferred to the agent. --- + ### Agent settings sources {: #agent-settings-sources} -Regardless of the source of the agent settings, the JSON document is the same. +Regardless of the source of the agent settings, the JSON document is the same. The agent settings as described above are transferred to the agent using one of the following ways: -* Metadata server is used if all involved components (director, CPI, agent) support CPI API V2. -* If one of the components does not yet support CPI API V2, the agent reads its +- Metadata server is used if all involved components (director, CPI, agent) support CPI API V2. +- If one of the components does not yet support CPI API V2, the agent reads its settings from the registry. !!! tip See [CPI API V2](cpi-api-v2.md) and [CPI V2 Migration Guide](cpi-api-v2-migration-guide.md) for more information on how the CPI, Agent, and Director behave in a registry-less environment. --- + ### CPI API V1 {: #cpi-api-v1} This section describes the agent interactions prior to CPI API V2 ----- -##### Metadata Server {: #metadata } +--- + +#### Metadata Server {: #metadata } The metadata server initially serves the registry URL and DNS server list. @@ -304,21 +313,22 @@ Following is a sample content of the user-data part of the HTTP metadata The supported format of the metadata server by the bosh-agent is documented in [UserDataContentsType](https://pkg.go.dev/github.com/cloudfoundry/bosh-agent/v2/infrastructure#UserDataContentsType) and [http\_metadata\_service_test.go](https://github.com/cloudfoundry/bosh-agent/blob/1dca3244702c18bf2c36483c529d4e7b3fb92b2e/infrastructure/http_metadata_service_test.go), along with the expected behavior of the bosh agent when reading this config. ----- -##### Registry {: #registry } +--- + +#### Registry {: #registry } The registry provides bosh-side metadata to the bosh agent. From the [Warden CPI documentation](https://github.com/cloudfoundry/bosh-warden-cpi-release/blob/be1869737c0bfba96662dde3499c9181863f91a7/docs/bosh-micro-usage.md): -* The registry is used by the CPI to pass data to the Agent. The registry is started on a server specified by registry properties. -* If SSH tunnel options are provided, a reverse ssh tunnel is created from the MicroBOSH VM to the registry, making the registry available to the agent on remote machine. +- The registry is used by the CPI to pass data to the Agent. The registry is started on a server specified by registry properties. +- If SSH tunnel options are provided, a reverse ssh tunnel is created from the MicroBOSH VM to the registry, making the registry available to the agent on remote machine. -###### Registry HTTP protocol {: #registry-protocol } +##### Registry HTTP protocol {: #registry-protocol } The Agent expects to communicate with the bosh registry over a REST API documented in [api\_controller\_spec.rb](https://github.com/cloudfoundry/bosh/blob/2f73281f1a2a155ee807e7c0c9b8187c5a742f78/bosh-registry/spec/unit/bosh/registry/api_controller_spec.rb) Reference registry client and servers implementations are available in: -* go-lang through [frodenas/bosh-registry](https://github.com/frodenas/bosh-registry) -* java through [cloudfoundry-community/bosh-cloudstack-cpi-core](https://github.com/cloudfoundry-community/bosh-cloudstack-cpi-core/tree/44d14a5f184d2d5e8f1f2fcd6344e734d3344673/src/main/java/com/orange/oss/cloudfoundry/cscpi/boshregistry) +- go-lang through [frodenas/bosh-registry](https://github.com/frodenas/bosh-registry) +- java through [cloudfoundry-community/bosh-cloudstack-cpi-core](https://github.com/cloudfoundry-community/bosh-cloudstack-cpi-core/tree/44d14a5f184d2d5e8f1f2fcd6344e734d3344673/src/main/java/com/orange/oss/cloudfoundry/cscpi/boshregistry) diff --git a/content/alicloud-cpi-errors.md b/content/alicloud-cpi-errors.md index b40a0abae..3122e78e1 100644 --- a/content/alicloud-cpi-errors.md +++ b/content/alicloud-cpi-errors.md @@ -1,3 +1,5 @@ +# Alicloud CPI Errors + ## Incorrect KeyPair > SDK.ServerError @@ -6,7 +8,6 @@ Make sure that your key_pair_name is correct and it is in the current region. - ## Incorrect Private IP > SDK.ServerError diff --git a/content/alicloud-cpi.md b/content/alicloud-cpi.md index feaecee45..d79e1a12e 100644 --- a/content/alicloud-cpi.md +++ b/content/alicloud-cpi.md @@ -1,10 +1,13 @@ +# AliCloud CPI Cloud Properties + This topic describes cloud properties for different resources created by the Alibaba Cloud CPI. ## AZs {: #azs } Schema for `cloud_properties` section: -* **availability_zone** [String, required]: Availability zone to use for creating instances. Example: `us-east-1a`. +- **availability_zone** [String, required]: Availability zone to use for creating instances. + - Example: `us-east-1a`. Example: @@ -16,12 +19,13 @@ azs: ``` --- + ## Networks {: #networks } Schema for `cloud_properties` section used by dynamic network or manual network subnet: -* **vswitch_id** [String, required]: VSwitch ID in which the instance will be created. Example: `vsw-123456abc`. -* **security_group_ids** [Array, optional]: Array of [Security Groups](https://www.alibabacloud.com/help/doc-detail/25468.htm), to apply to all VMs placed on this network. Security groups can be specified as follows, ordered by greatest precedence: `vm_types`, followed by `networks`. +- **vswitch_id** [String, required]: VSwitch ID in which the instance will be created. Example: `vsw-123456abc`. +- **security_group_ids** [Array, optional]: Array of [Security Groups](https://www.alibabacloud.com/help/doc-detail/25468.htm), to apply to all VMs placed on this network. Security groups can be specified as follows, ordered by greatest precedence: `vm_types`, followed by `networks`. Example of manual network: @@ -56,69 +60,78 @@ networks: ``` --- + ## VM Types / VM Extensions {: #resource-pools } Schema for `cloud_properties` section: -* **instance_type** [String, required]: Type of the [instance](https://www.alibabacloud.com/help/doc-detail/25378.htm). Example: `ecs.g5.large`. -* **availability_zone** [String, required]: Availability zone to use for creating instances. Example: `us-east-1a`. -* **security_group_ids** [Array, optional]: See description under [networks](#networks). Available in v19+. -* **key\_pair\_name** [String, optional]: Key pair name. Example: `bosh`. -* **spot\_price\_limit** [Float, optional]: Bid price in RMB for Alibaba Cloud spot instance. Using this option will slow down VM creation. Example: `0.03`. -* **spot_strategy** [String, optional]: Sets an expected spot price if you are creating preemptible instances. It takes effect only when parameter InstanceChargeType is PostPaid. Options: - - `NoSpot` A normal Pay-As-You-Go instance. - - `SpotWithPriceLimit` Sets the price threshold for a preemptible instance. - - `SpotAsPriceGo` A price that is based on the highest Pay-As-You-Go instance. - - Default value: NoSpot. - -* **slbs** [Array, optional]: Array of Load balancer Ids used to add created VMs to default slb server group. Example: `[lb-abc123456]`. Default is `[]`. -* **slb_weight** [Integer, optional]: The weight value used to add created VMs to default slb server group. Example: `80`. Default is `100`. -* **slb_server_group** [Array, optional]: Array of SLB virtual server group Ids used to attach created VMs. Example: `[sgp-abc123456]`. Default is `[]`. -* **slb_server_group_port** [Integer, optional]: The port value used to attach created VMs with SLB virtual server group. It will be ignored when `slb_server_group` is not set. Example: `8080`. Default is `33333`. -* **slb_server_group_weight** [Integer, optional]: The weight value used to attach created VMs with SLB virtual server group. It will be ignored when `slb_server_group` is not set. Example: `80`. Default is `100`. -* **nlb_server_group_ids** [Array, optional, Available since v47.0.0]: Array of NLB virtual server group Ids used to attach created VMs. Example: `[rgp-abc123456]`. Default is `[]`. -* **nlb_server_group_port** [Integer, optional, Available since v47.0.0]: The port value used to attach created VMs with NLB virtual server group. It will be ignored when `nlb_server_group_ids` is not set. Example: `8080`. Default is ``. -* **nlb_server_group_weight** [Integer, optional, Available since v47.0.0]: The weight value used to attach created VMs with NLB virtual server group. It will be ignored when `nlb_server_group_ids` is not set. Example: `80`. Default is `100`. -* * **ram\_role\_name** [String, optional]: Instance RAM role name. The name is provided and maintained by RAM and can be queried using [ListRoles](https://www.alibabacloud.com/help/doc-detail/28713.htm). - For more information, see [CreateRole](https://www.alibabacloud.com/help/doc-detail/28710.htm) and [ListRoles](https://www.alibabacloud.com/help/doc-detail/28713.htm). Example: `director`. -* **instance_name** [String, optional]: The instance name. It is a string of 2 to 128 English letters and special characters. It must begin with an English or a Chinese character. -It can contain digits, periods (.), colons (:), underscores (_), and hyphens (-), but cannot begin with http:// or https://. The default value is the `InstanceId` of the instance. -* **passowrd** [String, optional]: Password of the ECS instance. It can be [8, 30] characters in length. It must contain uppercase and lowercase letters, digits. The following special characters are allowed: @ # $ % ^ & * - + = | { } [ ] : ; ‘ < > , . ? / -* **charge_type** [String, optional]: Billing methods. Optional values: - - `PrePaid` Monthly, or annual subscription. Make sure that your registered credit card is invalid or you have insufficient balance in your PayPal account. Otherwise, InvalidPayMethod error may occur. - - `PostPaid` Pay-As-You-Go. - - Default value: PostPaid. - -* **charge_period** [Integer, optional]: The charge period of `PrePaid` instance. The value depends on `charge_period_unit`. -* **charge\_period\_unit** [String, optional]: The charge period unit of `PrePaid` instance. Optional values: Week | Month. When PeriodUnit is Week, period can be one of {“1”, “2”, “3”, “4”}. -When PeriodUnit is Month, period can be one of { “1”, “2”, “3”, “4”, “5”, “6”, “7”, “8”, “9”, “12”, “24”, “36”,”48”,”60”}. Default value: Month. -* **auto_renew** [Boolean, optional]: Whether to set AutoRenew. This parameter is valid when InstanceChargeType is PrePaid. Optional values: - - `True` Enable automatic renewal. - - `False` Disable automatic renewal. - - Default value: false. -* **auto\_renew\_period** [Integer, optional]: When AutoRenew is set to True, this parameter is required. When PeriodUnit is Week, AutoRenewPeriod can be one of {“1”, “2”, “3”}. - When PeriodUnit is Month, AutoRenewPeriod can be one of {“1”, “2”, “3”, “6”, “12”}. - -* **region** [String, optional]: Alibaba Cloud region id. Example: `us-east-1`. Available in v19+. Defaults to region specified by `region` in global CPI settings. -* **stemcell_id** [String, optional]: The specified stemcell id used to launch instance. It can be used to cross-region deployment. Available in v19+. - -* **ephemeral_disk** [Hash, optional]: Elastic block storage data disk of custom size. Default disk size is either the size of first instance storage disk. - * **size** [Integer, required]: Specifies the disk size in megabytes. - * **category** [String, optional]: category of the [Elastic block storage](https://www.alibabacloud.com/help/doc-detail/25383.htm): `cloud_efficiency`, `cloud_ssd`. Defaults to `cloud_efficiency`. - - `cloud_efficiency` Ultra cloud disk. - - `cloud_ssd` Cloud SSD. - * **encrypted** [Boolean, optional] Enables encryption for the ephemeral disk. Defaults to `false`. Overrides the global `encrypted` property. - * **delete\_with\_instance** [Boolean, optional] Whether a data disk is released along with the instance or not. Optional values: - - `true` The disk is released with the instance. - - `false` The disk is not released with the instance. -* **system_disk** [Hash, optional]: Elastic block storage system disk of custom size. - * **size** [Integer, required]: Specifies the disk size in megabytes. - * **category** [String, optional]: category of the [Elastic block storage](https://www.alibabacloud.com/help/doc-detail/25383.htm): `cloud_efficiency`, `cloud_ssd`. Defaults to `cloud_efficiency`. +- **instance_type** [String, required]: Type of the [instance](https://www.alibabacloud.com/help/doc-detail/25378.htm). + - Example: `ecs.g5.large`. +- **availability_zone** [String, required]: Availability zone to use for creating instances. + - Example: `us-east-1a`. +- **security_group_ids** [Array, optional]: See description under [networks](#networks). Available in v19+. +- **key\_pair\_name** [String, optional]: Key pair name. + - Example: `bosh`. +- **spot\_price\_limit** [Float, optional]: Bid price in RMB for Alibaba Cloud spot instance. Using this option will slow down VM creation. + - Example: `0.03`. +- **spot_strategy** [String, optional]: Sets an expected spot price if you are creating preemptible instances. It takes effect only when parameter InstanceChargeType is PostPaid. Options: + - `NoSpot` A normal Pay-As-You-Go instance. + - `SpotWithPriceLimit` Sets the price threshold for a preemptible instance. + - `SpotAsPriceGo` A price that is based on the highest Pay-As-You-Go instance. + - Default is `NoSpot`. +- **slbs** [Array, optional]: Array of Load balancer Ids used to add created VMs to default slb server group. Example: `[lb-abc123456]`. + - Default is `[]`. +- **slb_weight** [Integer, optional]: The weight value used to add created VMs to default slb server group. Example: `80`. + - Default is `100`. +- **slb_server_group** [Array, optional]: Array of SLB virtual server group Ids used to attach created VMs. Example: `[sgp-abc123456]`. + - Default is `[]`. +- **slb_server_group_port** [Integer, optional]: The port value used to attach created VMs with SLB virtual server group. It will be ignored when `slb_server_group` is not set. Example: `8080`. + - Default is `33333`. +- **slb_server_group_weight** [Integer, optional]: The weight value used to attach created VMs with SLB virtual server group. It will be ignored when `slb_server_group` is not set. Example: `80`. + - Default is `100`. +- **nlb_server_group_ids** [Array, optional, Available since v47.0.0]: Array of NLB virtual server group Ids used to attach created VMs. Example: `[rgp-abc123456]`. + - Default is `[]`. +- **nlb_server_group_port** [Integer, optional, Available since v47.0.0]: The port value used to attach created VMs with NLB virtual server group. It will be ignored when `nlb_server_group_ids` is not set. Example: `8080`. + - Default is \```\`. +- **nlb_server_group_weight** [Integer, optional, Available since v47.0.0]: The weight value used to attach created VMs with NLB virtual server group. It will be ignored when `nlb_server_group_ids` is not set. Example: `80`. + - Default is `100`. +- **ram\_role\_name** [String, optional]: Instance RAM role name. The name is provided and maintained by RAM and can be queried using [ListRoles](https://www.alibabacloud.com/help/doc-detail/28713.htm). +For more information, see [CreateRole](https://www.alibabacloud.com/help/doc-detail/28710.htm) and [ListRoles](https://www.alibabacloud.com/help/doc-detail/28713.htm). +Example: `director`. +- **instance_name** [String, optional]: The instance name. It is a string of 2 to 128 English letters and special characters. It must begin with an English or a Chinese character. It can contain digits, periods (.), colons (:), underscores (_), and hyphens (-), but cannot begin with http:// or https://. + - The default value is the `InstanceId` of the instance. +- **passowrd** [String, optional]: Password of the ECS instance. It can be [8, 30] characters in length. It must contain uppercase and lowercase letters, digits. The following special characters are allowed: @ # $ % ^ & * - + = | { } [ ] : ; ‘ < > , . ? / +- **charge_type** [String, optional]: Billing methods. Optional values: + - `PrePaid` Monthly, or annual subscription. Make sure that your registered credit card is invalid or you have insufficient balance in your PayPal account. Otherwise, InvalidPayMethod error may occur. + - `PostPaid` Pay-As-You-Go. + - Default is `PostPaid`. + +- **charge_period** [Integer, optional]: The charge period of `PrePaid` instance. The value depends on `charge_period_unit`. +- **charge\_period\_unit** [String, optional]: The charge period unit of `PrePaid` instance. Optional values: Week | Month. When PeriodUnit is Week, period can be one of {“1”, “2”, “3”, “4”}. When PeriodUnit is Month, period can be one of { “1”, “2”, “3”, “4”, “5”, “6”, “7”, “8”, “9”, “12”, “24”, “36”,”48”,”60”}. + - Default is `Month`. +- **auto_renew** [Boolean, optional]: Whether to set AutoRenew. This parameter is valid when InstanceChargeType is PrePaid. Optional values: + - `true` Enable automatic renewal. + - `false` Disable automatic renewal. + - Default is `false`. +- **auto\_renew\_period** [Integer, optional]: When AutoRenew is set to True, this parameter is required. When PeriodUnit is Week, AutoRenewPeriod can be one of {“1”, “2”, “3”}. When PeriodUnit is Month, AutoRenewPeriod can be one of {“1”, “2”, “3”, “6”, “12”}. + +- **region** [String, optional]: Alibaba Cloud region id. Example: `us-east-1`. Available in v19+. + - Defaults to region specified by `region` in global CPI settings. +- **stemcell_id** [String, optional]: The specified stemcell id used to launch instance. It can be used to cross-region deployment. Available in v19+. + +- **ephemeral_disk** [Hash, optional]: Elastic block storage data disk of custom size. + - Default disk size is either the size of first instance storage disk. + - **size** [Integer, required]: Specifies the disk size in megabytes. + - **category** [String, optional]: category of the [Elastic block storage](https://www.alibabacloud.com/help/doc-detail/25383.htm): `cloud_efficiency`, `cloud_ssd`. - `cloud_efficiency` Ultra cloud disk. - `cloud_ssd` Cloud SSD. + - Default is `cloud_efficiency`. + - **encrypted** [Boolean, optional] Enables encryption for the ephemeral disk. Defaults to `false`. Overrides the global `encrypted` property. + - **delete\_with\_instance** [Boolean, optional] Whether a data disk is released along with the instance or not. Optional values: + - `true` The disk is released with the instance. + - `false` The disk is not released with the instance. +- **system_disk** [Hash, optional]: Elastic block storage system disk of custom size. + - **size** [Integer, required]: Specifies the disk size in megabytes. Example of an `ecs.g5.large` instance: @@ -135,18 +148,20 @@ resource_pools: ``` --- + ## Disk Types {: #disk-pools } Schema for `cloud_properties` section: -* **category** [String, optional]: category of the [Elastic block storage](https://www.alibabacloud.com/help/doc-detail/25383.htm): `cloud_efficiency`, `cloud_ssd`. Defaults to `cloud_efficiency`. +- **category** [String, optional]: category of the [Elastic block storage](https://www.alibabacloud.com/help/doc-detail/25383.htm): `cloud_efficiency`, `cloud_ssd`. - `cloud_efficiency` Ultra cloud disk. - `cloud_ssd` Cloud SSD. - -* **encrypted** [Boolean, optional] Enables encryption for the ephemeral disk. Defaults to `false`. Overrides the global `encrypted` property. -* **delete\_with\_instance** [Boolean, optional] Whether a data disk is released along with the instance or not. Optional values: - - `true` The disk is released with the instance. - - `false` The disk is not released with the instance. + - Default is `cloud_efficiency`. +- **encrypted** [Boolean, optional] Enables encryption for the ephemeral disk. Overrides the global `encrypted` property. + - Default is `false`. +- **delete\_with\_instance** [Boolean, optional] Whether a data disk is released along with the instance or not. Optional values: + - `true` The disk is released with the instance. + - `false` The disk is not released with the instance. Elastic block storage volumes are created in the availability zone of an instance that volume will be attached. @@ -161,18 +176,25 @@ disk_pools: ``` --- + ## Global Configuration {: #global } The CPI can only talk to a single Alibaba Cloud region. Schema: -* **access\_key\_id** [String, optional]: Access Key ID. Example: `AKI...`. -* **access\_key\_secret** [String, optional]: Access Key Secret. Example: `0kwh...`. -* **security_token** [String, optional]: AliCloud [Security Token Service](https://www.alibabacloud.com/help/doc-detail/66222.html). Example: `0nwicsere...`. -* **region** [String, required]: Alibaba Cloud region name. Example: `us-east-1` -* **availability_zone** [String, required]: Availability zone to use for creating instances. Example: `us-east-1a`. -* **encrypted** [Boolean, optional]: Turns on [ECS disk encryption](https://www.alibabacloud.com/help/doc-detail/59643.htm) for all VM's data disks. Defaults to `false`. +- **access\_key\_id** [String, optional]: Access Key ID. + - Example: `AKI...`. +- **access\_key\_secret** [String, optional]: Access Key Secret. + - Example: `0kwh...`. +- **security_token** [String, optional]: AliCloud [Security Token Service](https://www.alibabacloud.com/help/doc-detail/66222.html). + - Example: `0nwicsere...`. +- **region** [String, required]: Alibaba Cloud region name. + - Example: `us-east-1` +- **availability_zone** [String, required]: Availability zone to use for creating instances. + - Example: `us-east-1a`. +- **encrypted** [Boolean, optional]: Turns on [ECS disk encryption](https://www.alibabacloud.com/help/doc-detail/59643.htm) for all VM's data disks. + - Defaults is `false`. Example with hard-coded credentials: @@ -185,6 +207,7 @@ properties: ``` --- + ## Example Cloud Config {: #cloud-config } ```yaml diff --git a/content/aws-cpi-errors.md b/content/aws-cpi-errors.md index 9011f3cd0..48cf80d03 100644 --- a/content/aws-cpi-errors.md +++ b/content/aws-cpi-errors.md @@ -1,45 +1,41 @@ +# AWS CPI Errors + ## Incorrect Region > Stemcell does not contain an AMI for this region (us-west-2c) Make sure that your [`region`](aws-cpi.md#options-region) is one of the [official AWS regions](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-regions). AWS regions typically end with a number, so in the example above region is erroneously specified (it's set to an AZ since each region is divided into multiple AZ which end with a letter.) - ## Elastic IP Requires an Internet Gateway > Network vpc-a09e18c5 is not attached to any internet gateway You need to create and attach an internet gateway to your VPC so that VMs can connect to the Internet. - ## Incorrect Subnet > The subnet ID 'subnet-c3051fad' does not exist Make sure that the [`region`](aws-cpi.md#options-region) matches the region where your specified subnet resides. - ## Incorrect System Time > Signature expired: 20141106T010406Z is now earlier than 20141106T011252Z (20141106T011752Z - 5 min.) This error is usually caused by out-of-sync system time. Use `ntpdate` to sync the clock on the machine where BOSH CLI is run: `sudo ntpdate pool.ntp.org`. Alternatively make sure that `ntpd` is correctly configured and running. - ## Multiple VMs Using an Elastic IP > resource eipalloc-6a45950f is already associated with associate-id eipassoc-427beb26 This error indicates that elastic IP specified in the manifest to be associated to the VM is in use by another VM. Check AWS console and decide whether other VM should be deleted to make elastic IP available for use. - ## Missing Subnet > Specifying an IP address is only valid for VPC instances and thus requires a subnet in which to launch Make sure that each manual network subnet has `cloud_properties` key and its contents include `subnet` key with the AWS Subnet ID. (You may have accidentally specified `cloud_properties` on the network itself.) - ## Incorrect Arguments > Arguments are not correct @@ -49,35 +45,30 @@ This error may be raised when: * `instance_type` is missing from the compilation or one of the resource pools' `cloud_properties` section * the deployment job instance is not assigned a static IP - ## Address is In Use > Address 10.10.16.251 is in use. This error indicates that unknown VM took up the IP that the Director is trying to assign to a new VM. Either let the Director know to not use this IP by including it in the reserved section of a subnet in your manual network, or make that IP available by terminating the unknown VM. - ## Consistent Security Groups > When specifying a security group you must specify a group id for each item. Make sure all security groups in the CPI configuration and networks' `cloud_properties` sections are specified in the same format, as IDs (e.g. `sg-384fher`) or names (e.g. `cf-public`). - ## Insufficient IAM Permissions > You are not authorized to perform this operation. Encoded authorization failure message: vHU-KncL6Yo4pG5J9p... See [IAM instance profiles errors](aws-iam-instance-profiles.md#errors). - ## Unsupported Instance Type Virtualization > Non-Windows instances with a virtualization type of 'hvm' are currently not supported for this instance type. You cannot use HVM stemcells with certain instance types. Review which instance type is specified in a referenced resource pool. - ## API Throttling > AWS::EC2::Errors::RequestLimitExceeded Request limit exceeded. diff --git a/content/aws-cpi.md b/content/aws-cpi.md index 3ea9d17f4..3902d66d0 100644 --- a/content/aws-cpi.md +++ b/content/aws-cpi.md @@ -1,10 +1,12 @@ +# AWS CPI Cloud Properties + This topic describes cloud properties for different resources created by the AWS CPI. ## AZs {: #azs } Schema for `cloud_properties` section: -* **availability_zone** [String, required]: Availability zone to use for creating instances. Example: `us-east-1a`. +- **availability_zone** [String, required]: Availability zone to use for creating instances. Example: `us-east-1a`. Example: @@ -16,12 +18,13 @@ azs: ``` --- + ## Networks {: #networks } Schema for `cloud_properties` section used by dynamic network or manual network subnet: -* **subnet** [String, required]: Subnet ID in which the instance will be created. Example: `subnet-9be6c3f7`. -* **security_groups** [Array, optional]: Array of [Security Groups](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-network-security.html), by name or ID, to apply to all VMs placed on this network. Security groups can be specified as follows, ordered by greatest precedence: `vm_types`, followed by `networks`, followed by `default_security_groups`. +- **subnet** [String, required]: Subnet ID in which the instance will be created. Example: `subnet-9be6c3f7`. +- **security_groups** [Array, optional]: Array of [Security Groups](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-network-security.html), by name or ID, to apply to all VMs placed on this network. Security groups can be specified as follows, ordered by greatest precedence: `vm_types`, followed by `networks`, followed by `default_security_groups`. Example of manual network: @@ -56,60 +59,61 @@ networks: ``` --- + ## VM Types / VM Extensions {: #resource-pools } Schema for `cloud_properties` section: -* **instance_type** [String, required]: Type of the [instance](http://aws.amazon.com/ec2/instance-types/). Example: `m3.medium`. -* **availability_zone** [String, required]: Availability zone to use for creating instances. Example: `us-east-1a`. -* **security_groups** [Array, optional]: See description under [networks](#networks). Available in v46+. -* **key_name** [String, optional]: Key pair name. Defaults to key pair name specified by `default_key_name` in global CPI settings. Example: `bosh`. -* **spot\_bid\_price** [Float, optional]: Bid price in dollars for [AWS spot instance](http://aws.amazon.com/ec2/purchasing-options/spot-instances/). Using this option will slow down VM creation. Example: `0.03`. -* **spot\_ondemand\_fallback** [Boolean, optional]: Set to `true` to use an on demand instance if a spot instance is not available during VM creation. Defaults to `false`. Available in v36. -* **elbs** [Array, optional]: Array of [Elastic (Classic) Load Balancer (ELB)](https://aws.amazon.com/documentation/elastic-load-balancing/) names that should be attached to created VMs. Example: `[prod-elb]`. Default is `[]`. -* **lb\_target\_groups** [Array, optional]: Array of Load Balancer Target Groups to which created VMs should be attached. Target Groups can be used to link [Application Load Balancers (ALB)](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/introduction.html) and [Network Load Balancers (NLB)](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/introduction.html) to instances. Example: `[prod-group1, prod-group2]`. Default is `[]`. Available in v63 or newer. -* **iam\_instance\_profile** [String, optional]: Name of an [IAM instance profile](aws-iam-instance-profiles.md). Example: `director`. -* **placement_group** [String, optional]: Name of a [placement group](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/placement-groups.html). Example: `my-group`. -* **tenancy** [String, optional]: VM [tenancy](http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/dedicated-instance.html) configuration. Example: `dedicated`. Default is `default`. -* **auto\_assign\_public\_ip** [Boolean, optional]: Assigns a public IP address to the created VM. This IP is ephemeral and may change; use an [Elastic IP](networks.md#vip) instead for a persistent address. Defaults to `false`. Available in v55+. -* **advertised\_routes** [Array, optional]: Creates routes in an [AWS Route Table](http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/VPC_Route_Tables.html) with the created BOSH VM as the target. Requires IAM action `ec2:CreateRoute`, `ec2:DescribeRouteTables`, `ec2:ReplaceRoute`. - * **table\_id** [String, required]: ID of the route table in which to create the route (e.g. `rt-abcdef123`). - * **destination** [String, required]: Destination CIDR for the route. All traffic with a destination within this CIDR will be routed through the created BOSH VM. -* **raw\_instance\_storage** [Boolean, optional]: Exposes all available [instance storage via labeled disks](aws-instance-storage.md). Defaults to `false`. -* **source\_dest\_check** [Boolean, optional]: Specifies whether the instance must be the source or destination of any traffic it sends or receives. If set to `false`, the instance does *not* need to be the source or destination. Used for network address translation (NAT) boxes, frequently to communicate between VPCs. Defaults to `true`. Requires IAM action `ec2:ModifyInstanceAttribute`. Available in v59+. -* **ephemeral_disk** [Hash, optional]: EBS backed ephemeral disk of custom size. Default disk size is either the size of first instance storage disk, if the instance_type offers it, or 10GB. Before v53: Used EBS only if instance storage is not large enough or not available for selected instance type. - * **size** [Integer, required]: Specifies the disk size in megabytes. - * **type** [String, optional]: Type of the [disk](http://aws.amazon.com/ebs/details/): `standard`, `gp2`, `gp3`. Defaults to `gp3` (since [v88](https://github.com/cloudfoundry/bosh-aws-cpi-release/releases/tag/v88)). - * `standard` stands for EBS magnetic drives - * `gp2` stands for EBS general purpose drives (SSD) - * `gp3` stands for EBS next-generation general purpose drives (SSD) - * `io1` stands for EBS provisioned IOPS drives (SSD) - * **iops** [Integer, optional]: Specifies the number of I/O operations per second to provision for the drive. - * Only valid for `io1` and `gp3` type drive. - * Required when `io1` type drive is specified. - * Optional for `gp3`, defaults to AWS default iops count if iops < AWS default (added with [v93](https://github.com/cloudfoundry/bosh-aws-cpi-release/releases/tag/v93)). - * **throughput** [Integer, optional]: Specifies Throughput in MBp/s - * Only valid for `gp3` type drive. - * Optional, defaults to AWS default throughput if throughput < AWS default - * Added with [v93](https://github.com/cloudfoundry/bosh-aws-cpi-release/releases/tag/v93). - * **encrypted** [Boolean, optional] Enables encryption for the EBS backed ephemeral disk. An error is raised, if the `instance_type` does not support it. Since v53. Defaults to `false`. Overrides the global `encrypted` property. - * **kms\_key\_arn** [String, optional] The ARN of an Amazon KMS key to use when encrypting the disk. - * **use\_instance\_storage** [Boolean, optional] Forces the usage of instance storage as ephemeral disk backing. Will raise an error, if the used `instance_type` does not have instance storage. Cannot be combined with any other option under `ephemeral_disk` or with `raw_instance_storage`. Since v53. Defaults to `false`. -* **root_disk** [Hash, optional]: EBS backed root disk of custom size. - * **size** [Integer, required]: Specifies the disk size in megabytes. - * **type** [String, optional]: Type of the [disk](http://aws.amazon.com/ebs/details/): `standard`, `gp2`, `gp3`. Defaults to `gp3` (since [v88](https://github.com/cloudfoundry/bosh-aws-cpi-release/releases/tag/v88)). - * `standard` stands for EBS magnetic drives - * `gp2` stands for EBS general purpose drives (SSD) - * `gp3` stands for EBS next-generation general purpose drives (SSD) - * **iops** [Integer, optional]: Specifies the number of I/O operations per second to provision for the drive. - * Only valid for `io1` and `gp3` type drive. - * Required when `io1` type drive is specified. - * Optional for `gp3`, defaults to AWS default iops count if iops < AWS default (added with [v93](https://github.com/cloudfoundry/bosh-aws-cpi-release/releases/tag/v93)). - * **throughput** [Integer, optional]: Specifies Throughput in MBp/s - * Only valid for `gp3` type drive. - * Optional, defaults to AWS default throughput if throughput < AWS default - * Added with [v93](https://github.com/cloudfoundry/bosh-aws-cpi-release/releases/tag/v93). -* **metadata_options** [Hash, optional]: Metadata configuration options that are set on a VM during creation. These options should be snake-cased properties accepted by the [ModifyInstanceMetadataOptions endpoint](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_ModifyInstanceMetadataOptions.html). e.g. `http_put_response_hop_limit`. If `metadata_options` is configured both at the global CPI level and as a VM extension, the VM extension properties take precedence. Available in v91+. +- **instance_type** [String, required]: Type of the [instance](http://aws.amazon.com/ec2/instance-types/). Example: `m3.medium`. +- **availability_zone** [String, required]: Availability zone to use for creating instances. Example: `us-east-1a`. +- **security_groups** [Array, optional]: See description under [networks](#networks). Available in v46+. +- **key_name** [String, optional]: Key pair name. Defaults to key pair name specified by `default_key_name` in global CPI settings. Example: `bosh`. +- **spot\_bid\_price** [Float, optional]: Bid price in dollars for [AWS spot instance](http://aws.amazon.com/ec2/purchasing-options/spot-instances/). Using this option will slow down VM creation. Example: `0.03`. +- **spot\_ondemand\_fallback** [Boolean, optional]: Set to `true` to use an on demand instance if a spot instance is not available during VM creation. Defaults to `false`. Available in v36. +- **elbs** [Array, optional]: Array of [Elastic (Classic) Load Balancer (ELB)](https://aws.amazon.com/documentation/elastic-load-balancing/) names that should be attached to created VMs. Example: `[prod-elb]`. Default is `[]`. +- **lb\_target\_groups** [Array, optional]: Array of Load Balancer Target Groups to which created VMs should be attached. Target Groups can be used to link [Application Load Balancers (ALB)](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/introduction.html) and [Network Load Balancers (NLB)](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/introduction.html) to instances. Example: `[prod-group1, prod-group2]`. Default is `[]`. Available in v63 or newer. +- **iam\_instance\_profile** [String, optional]: Name of an [IAM instance profile](aws-iam-instance-profiles.md). Example: `director`. +- **placement_group** [String, optional]: Name of a [placement group](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/placement-groups.html). Example: `my-group`. +- **tenancy** [String, optional]: VM [tenancy](http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/dedicated-instance.html) configuration. Example: `dedicated`. Default is `default`. +- **auto\_assign\_public\_ip** [Boolean, optional]: Assigns a public IP address to the created VM. This IP is ephemeral and may change; use an [Elastic IP](networks.md#vip) instead for a persistent address. Defaults to `false`. Available in v55+. +- **advertised\_routes** [Array, optional]: Creates routes in an [AWS Route Table](http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/VPC_Route_Tables.html) with the created BOSH VM as the target. Requires IAM action `ec2:CreateRoute`, `ec2:DescribeRouteTables`, `ec2:ReplaceRoute`. + - **table\_id** [String, required]: ID of the route table in which to create the route (e.g. `rt-abcdef123`). + - **destination** [String, required]: Destination CIDR for the route. All traffic with a destination within this CIDR will be routed through the created BOSH VM. +- **raw\_instance\_storage** [Boolean, optional]: Exposes all available [instance storage via labeled disks](aws-instance-storage.md). Defaults to `false`. +- **source\_dest\_check** [Boolean, optional]: Specifies whether the instance must be the source or destination of any traffic it sends or receives. If set to `false`, the instance does *not* need to be the source or destination. Used for network address translation (NAT) boxes, frequently to communicate between VPCs. Defaults to `true`. Requires IAM action `ec2:ModifyInstanceAttribute`. Available in v59+. +- **ephemeral_disk** [Hash, optional]: EBS backed ephemeral disk of custom size. Default disk size is either the size of first instance storage disk, if the instance_type offers it, or 10GB. Before v53: Used EBS only if instance storage is not large enough or not available for selected instance type. + - **size** [Integer, required]: Specifies the disk size in megabytes. + - **type** [String, optional]: Type of the [disk](http://aws.amazon.com/ebs/details/): `standard`, `gp2`, `gp3`. Defaults to `gp3` (since [v88](https://github.com/cloudfoundry/bosh-aws-cpi-release/releases/tag/v88)). + - `standard` stands for EBS magnetic drives + - `gp2` stands for EBS general purpose drives (SSD) + - `gp3` stands for EBS next-generation general purpose drives (SSD) + - `io1` stands for EBS provisioned IOPS drives (SSD) + - **iops** [Integer, optional]: Specifies the number of I/O operations per second to provision for the drive. + - Only valid for `io1` and `gp3` type drive. + - Required when `io1` type drive is specified. + - Optional for `gp3`, defaults to AWS default iops count if iops < AWS default (added with [v93](https://github.com/cloudfoundry/bosh-aws-cpi-release/releases/tag/v93)). + - **throughput** [Integer, optional]: Specifies Throughput in MBp/s + - Only valid for `gp3` type drive. + - Optional, defaults to AWS default throughput if throughput < AWS default + - Added with [v93](https://github.com/cloudfoundry/bosh-aws-cpi-release/releases/tag/v93). + - **encrypted** [Boolean, optional] Enables encryption for the EBS backed ephemeral disk. An error is raised, if the `instance_type` does not support it. Since v53. Defaults to `false`. Overrides the global `encrypted` property. + - **kms\_key\_arn** [String, optional] The ARN of an Amazon KMS key to use when encrypting the disk. + - **use\_instance\_storage** [Boolean, optional] Forces the usage of instance storage as ephemeral disk backing. Will raise an error, if the used `instance_type` does not have instance storage. Cannot be combined with any other option under `ephemeral_disk` or with `raw_instance_storage`. Since v53. Defaults to `false`. +- **root_disk** [Hash, optional]: EBS backed root disk of custom size. + - **size** [Integer, required]: Specifies the disk size in megabytes. + - **type** [String, optional]: Type of the [disk](http://aws.amazon.com/ebs/details/): `standard`, `gp2`, `gp3`. Defaults to `gp3` (since [v88](https://github.com/cloudfoundry/bosh-aws-cpi-release/releases/tag/v88)). + - `standard` stands for EBS magnetic drives + - `gp2` stands for EBS general purpose drives (SSD) + - `gp3` stands for EBS next-generation general purpose drives (SSD) + - **iops** [Integer, optional]: Specifies the number of I/O operations per second to provision for the drive. + - Only valid for `io1` and `gp3` type drive. + - Required when `io1` type drive is specified. + - Optional for `gp3`, defaults to AWS default iops count if iops < AWS default (added with [v93](https://github.com/cloudfoundry/bosh-aws-cpi-release/releases/tag/v93)). + - **throughput** [Integer, optional]: Specifies Throughput in MBp/s + - Only valid for `gp3` type drive. + - Optional, defaults to AWS default throughput if throughput < AWS default + - Added with [v93](https://github.com/cloudfoundry/bosh-aws-cpi-release/releases/tag/v93). +- **metadata_options** [Hash, optional]: Metadata configuration options that are set on a VM during creation. These options should be snake-cased properties accepted by the [ModifyInstanceMetadataOptions endpoint](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_ModifyInstanceMetadataOptions.html). e.g. `http_put_response_hop_limit`. If `metadata_options` is configured both at the global CPI level and as a VM extension, the VM extension properties take precedence. Available in v91+. Example of an `m3.medium` instance: @@ -126,25 +130,26 @@ resource_pools: ``` --- + ## Disk Types {: #disk-pools } Schema for `cloud_properties` section: -* **type** [String, optional]: Type of the [disk](http://aws.amazon.com/ebs/details/): `standard`, `gp2`, `gp3`. Defaults to `gp3` (since [v88](https://github.com/cloudfoundry/bosh-aws-cpi-release/releases/tag/v88)). - * `standard` stands for EBS magnetic drives - * `gp2` stands for EBS general purpose drives (SSD) - * `gp3` stands for EBS next-generation general purpose drives (SSD) - * `io1` stands for EBS provisioned IOPS drives (SSD) -* **iops** [Integer, optional]: Specifies the number of I/O operations per second to provision for the drive. - * Only valid for `io1` type drive. - * Required when `io1` type drive is specified. - * Optional for `gp3`, defaults to AWS default iops count if iops < AWS default (added with [v93](https://github.com/cloudfoundry/bosh-aws-cpi-release/releases/tag/v93)). -* **throughput** [Integer, optional]: Specifies Throughput in MBp/s - * Only valid for `gp3` type drive. - * Optional, defaults to AWS default throughput if throughput < AWS default - * Added with [v93](https://github.com/cloudfoundry/bosh-aws-cpi-release/releases/tag/v93). -* **encrypted** [Boolean, optional]: Turns on [EBS volume encryption](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSEncryption.html) for this persistent disk. VM root and ephemeral disk are not encrypted. Defaults to `false`. Overrides the global `encrypted` property. -* **kms\_key\_arn** [String, optional]: Encrypts the disk using an encryption key stored in the [AWS Key Management Service (KMS)](https://aws.amazon.com/kms/). The format of the ID is `XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX`. Be sure to use the Key ID, not the Alias. If omitted the disk will be encrypted using the global `kms_key_arn` property. If, no global `kms_key_arn` is set will use your account's default `aws/ebs` encryption key. +- **type** [String, optional]: Type of the [disk](http://aws.amazon.com/ebs/details/): `standard`, `gp2`, `gp3`. Defaults to `gp3` (since [v88](https://github.com/cloudfoundry/bosh-aws-cpi-release/releases/tag/v88)). + - `standard` stands for EBS magnetic drives + - `gp2` stands for EBS general purpose drives (SSD) + - `gp3` stands for EBS next-generation general purpose drives (SSD) + - `io1` stands for EBS provisioned IOPS drives (SSD) +- **iops** [Integer, optional]: Specifies the number of I/O operations per second to provision for the drive. + - Only valid for `io1` type drive. + - Required when `io1` type drive is specified. + - Optional for `gp3`, defaults to AWS default iops count if iops < AWS default (added with [v93](https://github.com/cloudfoundry/bosh-aws-cpi-release/releases/tag/v93)). +- **throughput** [Integer, optional]: Specifies Throughput in MBp/s + - Only valid for `gp3` type drive. + - Optional, defaults to AWS default throughput if throughput < AWS default + - Added with [v93](https://github.com/cloudfoundry/bosh-aws-cpi-release/releases/tag/v93). +- **encrypted** [Boolean, optional]: Turns on [EBS volume encryption](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSEncryption.html) for this persistent disk. VM root and ephemeral disk are not encrypted. Defaults to `false`. Overrides the global `encrypted` property. +- **kms\_key\_arn** [String, optional]: Encrypts the disk using an encryption key stored in the [AWS Key Management Service (KMS)](https://aws.amazon.com/kms/). The format of the ID is `XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX`. Be sure to use the Key ID, not the Alias. If omitted the disk will be encrypted using the global `kms_key_arn` property. If, no global `kms_key_arn` is set will use your account's default `aws/ebs` encryption key. EBS volumes are created in the availability zone of an instance that volume will be attached. @@ -158,28 +163,29 @@ Example of 10GB disk: ``` --- + ## Global Configuration {: #global } The CPI can only talk to a single AWS region. Schema: -* **credentials_source** [String, optional]: Selects credentials source between credentials provided in this configuration, or from an [IAM instance profile](aws-iam-instance-profiles.md). Default: `static`. -* **access\_key\_id** [String, optional]: Access Key ID. Example: `AKI...`. -* **secret\_access\_key** [String, optional]: Secret Access Key. Example: `0kwh...`. -* **default\_key\_name** [String, required]: Name of the [Key Pair](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html) that will be applied to all created VMs. Example: `bosh` -* **default\_security\_groups** [Array, required]: See description under [networks](#networks). -* **default\_iam\_instance\_profile** [String, optional]: Name of the [IAM instance profile](aws-iam-instance-profiles.md) that will be applied to all created VMs. Example: `director`. -* **region** [String, required]: AWS region name. Example: `us-east-1` -* **max_retries** [Integer, optional]: The maximum number of times AWS service errors (500) and throttling errors (`AWS::EC2::Errors::RequestLimitExceeded`) should be retried. There is an exponential backoff in between retries, so the more retries the longer it can take to fail. Defaults to 2. -* **encrypted** [Boolean, optional]: Turns on [EBS volume encryption](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSEncryption.html) for all VM's root (system), ephemeral and persistent disks. Defaults to `false`. Available in v67+. +- **credentials_source** [String, optional]: Selects credentials source between credentials provided in this configuration, or from an [IAM instance profile](aws-iam-instance-profiles.md). Default: `static`. +- **access\_key\_id** [String, optional]: Access Key ID. Example: `AKI...`. +- **secret\_access\_key** [String, optional]: Secret Access Key. Example: `0kwh...`. +- **default\_key\_name** [String, required]: Name of the [Key Pair](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html) that will be applied to all created VMs. Example: `bosh` +- **default\_security\_groups** [Array, required]: See description under [networks](#networks). +- **default\_iam\_instance\_profile** [String, optional]: Name of the [IAM instance profile](aws-iam-instance-profiles.md) that will be applied to all created VMs. Example: `director`. +- **region** [String, required]: AWS region name. Example: `us-east-1` +- **max_retries** [Integer, optional]: The maximum number of times AWS service errors (500) and throttling errors (`AWS::EC2::Errors::RequestLimitExceeded`) should be retried. There is an exponential backoff in between retries, so the more retries the longer it can take to fail. Defaults to 2. +- **encrypted** [Boolean, optional]: Turns on [EBS volume encryption](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSEncryption.html) for all VM's root (system), ephemeral and persistent disks. Defaults to `false`. Available in v67+. !!! warning EBS volume encryption does not work for Windows stemcells due to an [AWS limitation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/CopyingAMIs.html#copy-ami-across-accounts). Enabling this will not encrypt the root disk of Windows VMs. -* **kms\_key\_arn** [String, optional]: Encrypts the disks using an encryption key stored in the [AWS Key Management Service (KMS)](https://aws.amazon.com/kms/). The format of the ID is `XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX`. Be sure to use the Key ID, not the Alias. If this property is omitted and `encrypted` is `true`, the disks will be encrypted using your account's default `aws/ebs` encryption key. Available in v67+. -* **metadata_options** [Hash, optional]: Metadata configuration options that are set on a VM during creation. These options should be snake-cased properties accepted by the [ModifyInstanceMetadataOptions endpoint](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_ModifyInstanceMetadataOptions.html). e.g. `http_put_response_hop_limit`. If `metadata_options` is configured both at the global CPI level and as a VM extension, the VM extension properties take precedence. Available in v91+. -* **enable\_describe\_instance\_types** [Boolean, optional]: When `true` (default), the CPI calls the `ec2:DescribeInstanceTypes` API to accurately detect NVMe characteristics for EBS volume paths and raw instance storage device naming. Set to `false` to use a built-in static list of known instance families instead, which requires no `ec2:DescribeInstanceTypes` IAM permission but may not cover instance families released after this CPI version. Defaults to `true`. +- **kms\_key\_arn** [String, optional]: Encrypts the disks using an encryption key stored in the [AWS Key Management Service (KMS)](https://aws.amazon.com/kms/). The format of the ID is `XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX`. Be sure to use the Key ID, not the Alias. If this property is omitted and `encrypted` is `true`, the disks will be encrypted using your account's default `aws/ebs` encryption key. Available in v67+. +- **metadata_options** [Hash, optional]: Metadata configuration options that are set on a VM during creation. These options should be snake-cased properties accepted by the [ModifyInstanceMetadataOptions endpoint](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_ModifyInstanceMetadataOptions.html). e.g. `http_put_response_hop_limit`. If `metadata_options` is configured both at the global CPI level and as a VM extension, the VM extension properties take precedence. Available in v91+. +- **enable\_describe\_instance\_types** [Boolean, optional]: When `true` (default), the CPI calls the `ec2:DescribeInstanceTypes` API to accurately detect NVMe characteristics for EBS volume paths and raw instance storage device naming. Set to `false` to use a built-in static list of known instance families instead, which requires no `ec2:DescribeInstanceTypes` IAM permission but may not cover instance families released after this CPI version. Defaults to `true`. See [all configuration options](https://bosh.io/jobs/aws_cpi?source=github.com/cloudfoundry/bosh-aws-cpi-release). @@ -204,6 +210,7 @@ region: us-east-1 ``` --- + ## Example Cloud Config {: #cloud-config } ```yaml diff --git a/content/aws-iam-instance-profiles.md b/content/aws-iam-instance-profiles.md index 3927f2aae..c6acba747 100644 --- a/content/aws-iam-instance-profiles.md +++ b/content/aws-iam-instance-profiles.md @@ -1,3 +1,5 @@ +# AWS IAM Profiles + !!! note This feature is available with bosh-release v208+ (1.3087.0) colocated with bosh-aws-cpi v31+. @@ -41,6 +43,7 @@ You may have to create one or more IAM instance profiles to limit access to AWS Even though value specified is `env_or_profile`, `bosh create-env` command or the Director do not currently take advantage of the environment variables, only instance the profile, hence to take advantage of this feature you have to run on an AWS instance. --- + ## Example B: AWS CPI and Director configured with an S3 blobstore {: #director-with-s3-blobstore } This configuration is similar to the previous one except that it's used when the Director and the Agents use S3 as their [blobstore](bosh-components.md#blobstore) instead of an internal blobstore provided by the bosh release. @@ -95,9 +98,10 @@ This configuration is similar to the previous one except that it's used when the `iam_instance_profile` key in resource pool's cloud_properties takes precedence over the default IAM instance profile, so that specific VMs can have greater access to the AWS resources. --- + ## Errors {: #errors } -``` +```text You are not authorized to perform this operation. Encoded authorization failure message: vHU-KncL6Yo4pG5J9p... ``` diff --git a/content/aws-iam-users.md b/content/aws-iam-users.md index 9efd565f5..dc0121083 100644 --- a/content/aws-iam-users.md +++ b/content/aws-iam-users.md @@ -1,3 +1,5 @@ +# AWS IAM Users + ## Creating new IAM user {: #create } 1. Log into the AWS console: [https://console.aws.amazon.com/console/home](https://console.aws.amazon.com/console/home). diff --git a/content/aws-instance-storage.md b/content/aws-instance-storage.md index 3e3a65be6..01a56cba4 100644 --- a/content/aws-instance-storage.md +++ b/content/aws-instance-storage.md @@ -1,3 +1,5 @@ +# AWS - Using Instance Storage + !!! note This feature is available with bosh-aws-cpi v32+ and only for releases deployed with ? stemcells. diff --git a/content/aws.md b/content/aws.md index a702336cf..7da5b7aa7 100644 --- a/content/aws.md +++ b/content/aws.md @@ -1,38 +1,31 @@ ---- -title: Amazon Web Services ---- - # Amazon Web Services The `aws` CPI can be used with [Amazon Web Services](https://aws.amazon.com/). - * Release: [cloudfoundry/bosh-aws-cpi-release](https://github.com/cloudfoundry/bosh-aws-cpi-release) - * Issues: [GitHub Issues](https://github.com/cloudfoundry/bosh-aws-cpi-release/issues) - * Slack: [cloudfoundry#bosh](https://cloudfoundry.slack.com/messages/bosh) - +- Release: [cloudfoundry/bosh-aws-cpi-release](https://github.com/cloudfoundry/bosh-aws-cpi-release) +- Issues: [GitHub Issues](https://github.com/cloudfoundry/bosh-aws-cpi-release/issues) +- Slack: [cloudfoundry#bosh](https://cloudfoundry.slack.com/messages/bosh) ## Concepts The following table maps BOSH concepts to their AWS-native equivalents. -| BOSH | Amazon Web Services | -| ----------------- | ------------------- | +| BOSH | Amazon Web Services | +| ----------------- | ------------------- | | Availability Zone | [Availability Zone](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.RegionsAndAvailabilityZones.html) | -| Virtual Machine | [EC2 Instance](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Instances.html) | -| Network Subnet | [VPC Subnet](https://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/VPC_Subnets.html) | -| Virtual IP | [EC2 Elastic IP](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/elastic-ip-addresses-eip.html) | -| Persistent Disk | [EC2 EBS Volume](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumes.html) | -| Disk Snapshot | [EC2 EBS Snapshot](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSSnapshots.html) | -| Stemcell | [EC2 Amazon Machine Image](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) | -| Agent Settings | EC2 Instance User Metadata; BOSH Registry | - +| Virtual Machine | [EC2 Instance](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Instances.html) | +| Network Subnet | [VPC Subnet](https://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/VPC_Subnets.html) | +| Virtual IP | [EC2 Elastic IP](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/elastic-ip-addresses-eip.html) | +| Persistent Disk | [EC2 EBS Volume](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumes.html) | +| Disk Snapshot | [EC2 EBS Snapshot](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSSnapshots.html) | +| Stemcell | [EC2 Amazon Machine Image](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) | +| Agent Settings | EC2 Instance User Metadata; BOSH Registry | ## Feature Support The following sections describe some specific BOSH features supported by the CPI. - ### Network The CPI does not support multiple NICs being attached to a VM. @@ -43,7 +36,6 @@ The CPI does not support multiple NICs being attached to a VM. | Dynamic | Single network per instance | | VIP | Single network per instance | - ### Encryption AWS supports encryption functionality through their diff --git a/content/azs.md b/content/azs.md index fed163e6c..15ffaaae7 100644 --- a/content/azs.md +++ b/content/azs.md @@ -1,3 +1,5 @@ +# Availability Zones + First-class availability zones help expressing how VM instances, gathered into instance groups, will span over one or many availability zones. @@ -37,16 +39,13 @@ azs: AZs schema: -* **azs** [Array, required]: List of availability zones. +- **azs** [Array, required]: List of availability zones. -* **name** [String, required]: Name of an AZ within the Director. +- **name** [String, required]: Name of an AZ within the Director. -* **cloud_properties** [Hash, optional]: Describes any IaaS-specific - properties needed to associated with AZ; for most IaaSes, some data here is - actually required. - See [CPI Specific `cloud_properties`](#azs-cloud-properties) below. - Example: `availability_zone`. - Default is `{}` (empty Hash). +- **cloud_properties** [Hash, optional]: Describes any IaaS-specific properties needed to associated with AZ; for most IaaSes, some data here is actually required. See [CPI Specific `cloud_properties`](#azs-cloud-properties) below. + - Example: `availability_zone`. + - Default is `{}` (empty Hash). ### CPI Specific `cloud_properties` {: #azs-cloud-properties } @@ -138,9 +137,7 @@ VMs total: 4 are associated with a Region. This is important to keep in mind when planning an AZ migration. - !!! danger "General limitations" - - Bosh does not migrate persistent disk contents across availability zones. Persistent disks attached to an instance (VM) that is moved to another AZ will be orphaned and eventually deleted by Bosh. @@ -158,7 +155,7 @@ VMs total: 4 Given such a `dummy` instance group defined in a Bosh deployment manifest: -``` +```yaml instance_groups: - name: dummy azs: @@ -169,7 +166,7 @@ instance_groups: And Bosh instances (VMs) spread over the above availability zone like this: -``` +```shell $ bosh instances Instance ... AZ dummy/3697cb63-5329-4b61-8251-6acd73fe5d8b ... z3 @@ -180,7 +177,7 @@ dummy/7f30aae8-9f03-4b0b-88a1-2d0ab8a78fba ... z1 Then after adding the new `z2` availability zone to the instance group, the `bosh deploy` operation will behave as follows: -``` +```yaml Using deployment 'dummy' instance_groups: @@ -198,7 +195,7 @@ Updating instance dummy: dummy/e6764262-f032-4238-bfbe-d684934ece26 And after this operation is done, the resulting instances (VMs) will be placed like this: -``` +```shell $ bosh instances Instance ... AZ dummy/3697cb63-5329-4b61-8251-6acd73fe5d8b ... z3 @@ -221,20 +218,18 @@ certain considerations when spreading instances (VMs) across AZs: placement on availability zones will satisfy the IP addresses assignment on its availability zone. -Since the the above scenario **_does not_** utilize persistent disks, adding -`z2` to the `azs` list of availability zones will: - -1. Create a new instance (VM) in `z2` so that “_new instances will be spread - as evenly as possible over specified AZs_”. -2. Delete a currently serving instance (VM) in `z1` so that “_instances will - be rebalanced ... to even out distribution_” +Since the the above scenario ***does not*** utilize persistent disks, adding `z2` to the `azs` list of availability zones will: +1. Create a new instance (VM) in `z2` so that “*new instances will be spread + as evenly as possible over specified AZs*”. +2. Delete a currently serving instance (VM) in `z1` so that “*instances will + be rebalanced ... to even out distribution*” #### Scenario #2: the instance group uses persistent disks Given such a `dummy` instance group defined in a Bosh deployment manifest: -``` +```yaml instance_groups: - name: dummy azs: @@ -247,7 +242,7 @@ instance_groups: And Bosh instances (VMs) spread over the above availability zones like this: -``` +```shell $ bosh instances Instance ... AZ dummy/3697cb63-5329-4b61-8251-6acd73fe5d8b ... z3 @@ -258,7 +253,7 @@ dummy/528993ea-5e8b-4d7f-8844-98234bcb0575 ... z1 Then after adding the new `z2` availability zone to the instance group, the `bosh deploy` operation will behave as follows: -``` +```yaml Using deployment 'dummy' instance_groups: @@ -274,7 +269,7 @@ Updating instance dummy: dummy/3697cb63-5329-4b61-8251-6acd73fe5d8b (2) And after this is done, the resulting instances (VMs) will be placed like this: -``` +```shell $ bosh instances Instance ... AZ dummy/3697cb63-5329-4b61-8251-6acd73fe5d8b ... z3 @@ -300,7 +295,6 @@ prejudiciable data loss, depending on the deployed technology. preventing it, and still can recreate nodes with new disks when the deployment manifest instructs such changes. - ### Rebalancing instances with persistent disks Bosh's logic currently has a limitation in regards to balancing instances @@ -315,13 +309,13 @@ Details can be found in this Github issue: In the “[Assigning AZs](#assigning-azs)” section, the documentation outlines the considerations when spreading instances (VMs) across availability zones -when deploying. **_This does not fully apply when deleting instances_**. +when deploying. ***This does not fully apply when deleting instances***. #### Example Starting with three instances spread over two availability zones as follows: -``` +```shell $ bosh instances Instance ... AZ dummy/6488acf4-ea9d-4aab-aad5-95df06fc43a2 ... z1 @@ -332,7 +326,7 @@ dummy/b5b14411-f9ee-4ff8-95c6-b9c24b29b703 ... z1 Adding one instance and one availability zone, will result in such placement after `bosh deploy` has successfully finished: -``` +```shell $ bosh instances Instance ... AZ dummy/6488acf4-ea9d-4aab-aad5-95df06fc43a2 ... z1 @@ -343,7 +337,7 @@ dummy/3697cb63-5329-4b61-8251-6acd73fe5d8b ... z3 Removing one instance will result in such placement: -``` +```shell $ bosh instances Instance ... AZ dummy/6488acf4-ea9d-4aab-aad5-95df06fc43a2 ... z1 @@ -362,7 +356,6 @@ sections: 1. Automatic approach, resulting in a temporarily reduced instance count 2. Manual approach, but never going below the current number of available instances - #### Approach #1: automatic, but resulting in a temporarily reduced instance count If your distributed software supports syncing state between existing cluster @@ -375,7 +368,7 @@ add an availability zone, then scale out again. Given a `dummy` instance group resulting in Bosh instances (VMs) spread over two availability zones like this: -``` +```shell $ bosh instances Instance ... AZ dummy/6488acf4-ea9d-4aab-aad5-95df06fc43a2 ... z1 @@ -386,7 +379,7 @@ dummy/b5b14411-f9ee-4ff8-95c6-b9c24b29b703 ... z1 Reducing the instance count by one will result in such diff when converging the deployment with `bosh deploy`: -``` +```yaml Using deployment 'dummy' instance_groups: @@ -400,7 +393,7 @@ Using deployment 'dummy' And after the first `bosh deploy` operation has finished, the `dummy` instance group will have the exceeding instance placed in `z1` removed, as shown below. -``` +```shell $ bosh instances Instance ... AZ dummy/6488acf4-ea9d-4aab-aad5-95df06fc43a2 ... z1 @@ -412,7 +405,7 @@ dummy/6c002f9c-ab11-4468-9bcb-578819cf4b77 ... z2 Increasing again the instance count by `1` will result in such output when running `bosh deploy`: -``` +```yaml Using deployment 'dummy' instance_groups: @@ -429,7 +422,7 @@ instance group will look have the removed instance back, and it will be properly placed in `z3`, bringing back the expected balance in instance placement. -``` +```shell $ bosh instances Instance ... AZ dummy/6488acf4-ea9d-4aab-aad5-95df06fc43a2 ... z1 @@ -450,7 +443,7 @@ initial state. Given a `dummy` instance group resulting in Bosh instances (VMs) spread over two availability zones like this: -``` +```shell $ bosh instances Instance ... AZ dummy/6488acf4-ea9d-4aab-aad5-95df06fc43a2 ... z1 @@ -461,7 +454,7 @@ dummy/7e433b3e-2db8-46bf-883a-1c5300dfe104 ... z1 Introducing a new `z3` availability zone and increasing the instance count by one at the same time will result in the following `bosh deploy` task log: -``` +```yaml instance_groups: - name: dummy azs: @@ -477,7 +470,7 @@ Updating instance dummy: dummy/93fd5c41-88e2-4b2f-97ae-b064d507f3d5 The resulting instances will be placed as shown below. -``` +```shell $ bosh instances Instance ... AZ dummy/6488acf4-ea9d-4aab-aad5-95df06fc43a2 ... z1 @@ -504,7 +497,7 @@ remove the instance from the deployment state. It only deletes the related VM on the infrastructure and marks the instance in its internal representation with a sticky “stopped” state. -``` +```shell Task 70345 | 13:09:03 | Updating instance dummy: dummy/7e433b3e-2db8-46bf-883a-1c5300dfe104 (3) Task 70345 | 13:09:03 | L executing pre-stop: dummy/7e433b3e-2db8-46bf-883a-1c5300dfe104 (3) Task 70345 | 13:09:03 | L executing drain: dummy/7e433b3e-2db8-46bf-883a-1c5300dfe104 (3) @@ -515,7 +508,7 @@ Task 70345 | 13:09:05 | L executing post-stop: dummy/7e433b3e-2db8-46bf-883a-1c5 The scale-in operation will produce an orphaned disk from the deleted stateful instance, which can be listed as follows. -``` +```shell $ bosh disks --orphaned | grep 'dummy/7e433b3e-2db8-46bf-883a-1c5300dfe104' disk-147a80e4-72b0-4d77-7325-af28ae469d36 1.0 GiB dummy dummy/7e433b3e-2db8-46bf-883a-1c5300dfe104 z1 Fri Nov 18 13:12:12 UTC 2022 ``` @@ -523,7 +516,7 @@ disk-147a80e4-72b0-4d77-7325-af28ae469d36 1.0 GiB dummy dummy/7e433b3e-2 And after the `bosh stop` operation has finished, the `dummy` instance group will have the exceeding instance placed in `z1` removed, as shown below. -``` +```shell $ bosh instances Instance ... AZ dummy/6488acf4-ea9d-4aab-aad5-95df06fc43a2 ... z1 @@ -540,7 +533,7 @@ infrastructure, matching the required instances in the deployment. As a consequence, Bosh only deletes the reference to the instance that was just stopped. -``` +```yaml Using deployment 'dummy' instance_groups: @@ -552,7 +545,6 @@ Deleting unneeded instances dummy: dummy/7e433b3e-2db8-46bf-883a-1c5300dfe104 (3 ... ``` - ### Removing an AZ from an existing deployment When decommissioning an availability zone (AZ) by removing it from the @@ -565,7 +557,7 @@ Any new instance created afterwards will be balanced to the remaining AZs. Considering a `dummy` instance group with 3 instances placed in 3 availability zones as follows: -``` +```shell $ bosh instances Instance ... AZ dummy/6488acf4-ea9d-4aab-aad5-95df06fc43a2 ... z1 @@ -576,7 +568,7 @@ dummy/93fd5c41-88e2-4b2f-97ae-b064d507f3d5 ... z3 When removing one of the AZs and keeping the same instance count, the `bosh deploy` operation will show the following output: -``` +```yaml Using deployment 'dummy' instance_groups: @@ -592,7 +584,7 @@ Deleting unneeded instances dummy: dummy/93fd5c41-88e2-4b2f-97ae-b064d507f3d5 (2 After the above operation is done, the instances will be placed as follows. -``` +```shell $ bosh instances Instance ... AZ dummy/6488acf4-ea9d-4aab-aad5-95df06fc43a2 ... z1 @@ -603,12 +595,11 @@ dummy/c53e18df-1e47-44f9-9e41-3ee999aa4a87 ... z1 And while the new instance in `z1` will get an empty new persistent disk, such an orphaned disk will result from the former instance in `z3` being deleted: -``` +```shell bosh disks --orphaned | grep dummy/93fd5c41-88e2-4b2f-97ae-b064d507f3d5 disk-ce15e36a-1eeb-45da-494a-7282a56f3b32 1.0 GiB dummy dummy/93fd5c41-88e2-4b2f-97ae-b064d507f3d5 z3 Fri Nov 18 13:55:17 UTC 2022 ``` - ### Replacing an AZ in an existing deployment When replacing an AZ with another, Bosh will delete all existing instances @@ -617,7 +608,7 @@ will be orphaned. Replacement instances will be balanced into all AZs. !!! Warning Removing an availability zone (AZ) will delete all related instances (VMs) - _at the same time_. Even if you may have specified `canaries: 1` and + *at the same time*. Even if you may have specified `canaries: 1` and `max_in_flight: 1` in some applicable `update` block of your deployment manifest, all instances will be deleted in parallel. @@ -626,7 +617,7 @@ will be orphaned. Replacement instances will be balanced into all AZs. Given a `dummy` instance group resulting in Bosh instances (VMs) spread over two availability zones like this: -``` +```shell $ bosh instances Instance ... AZ dummy/6488acf4-ea9d-4aab-aad5-95df06fc43a2 ... z1 @@ -636,7 +627,7 @@ dummy/c53e18df-1e47-44f9-9e41-3ee999aa4a87 ... z1 The `bosh deploy` operation will output such task logs: -``` +```yaml instance_groups: - name: dummy azs: @@ -656,7 +647,7 @@ Updating instance dummy: dummy/4a1840a7-b239-4635-9d8e-1830567cd040 And the resulting instances will be placed on `z2` and `z3` like this: -``` +```shell Instance ... AZ dummy/4a1840a7-b239-4635-9d8e-1830567cd040 ... z3 dummy/6c002f9c-ab11-4468-9bcb-578819cf4b77 ... z2 @@ -665,19 +656,19 @@ dummy/cbb84b42-e6a6-4b4d-b560-e418177d2d6f ... z2 ## Migrating from Bosh v1 to Bosh v2 first-class AZs -Previously with “_Bosh v1_” deployment manifests, to spread resources over +Previously with “*Bosh v1*” deployment manifests, to spread resources over multiple availability zones (AZs), deployment jobs, resource pools, and networks had to be duplicated and named differently in the deployment manifest. By convention, all of these resources were suffixed with `_z1` or `zX` to indicate which AZ they belonged to. -With first-class AZs support in the Director, “_Bosh v2_” deployment manifests +With first-class AZs support in the Director, “*Bosh v2*” deployment manifests no longer need to duplicate and rename resources. This allows the Director to eliminate and/or simplify manual configuration for balancing instances (VMs) across AZs and IP address management. !!! Caveat - From a “_Bosh v1_” director, once you opt into using the “_Bosh v2_” Cloud + From a “*Bosh v1*” director, once you opt into using the “*Bosh v2*” Cloud Config, all deployments must be converted to use new format. There is no - way back to “_Bosh v1_” deployment manifests after you've opted in to + way back to “*Bosh v1*” deployment manifests after you've opted in to Cloud Config. diff --git a/content/azure-compute-gallery.md b/content/azure-compute-gallery.md index d8af1f5c2..aa224b91b 100644 --- a/content/azure-compute-gallery.md +++ b/content/azure-compute-gallery.md @@ -1,3 +1,5 @@ +# Azure Compute Gallery + [Azure Compute Gallery][azure-compute-gallery] allows you to manage, share, and distribute VM images across multiple regions and subscriptions within Azure. The Azure Compute Gallery can be used in the context of BOSH to store [BOSH stemcell](./stemcell.md) VHDs as Compute Gallery images. When deploying VMs with Compute Gallery enabled, BOSH automatically selects the appropriate image based on the stemcell configuration in the deployment manifest. Using Azure Compute Gallery Images in BOSH offers several benefits: @@ -14,10 +16,10 @@ Before you begin, ensure you have: - Azure Subscription with permissions to create and manage Azure Compute Galleries. - Azure Service Principal (configured in the [CPI global configuration](./azure-cpi.md#global)) with the following minimal roles assigned: -| **Role (Type)** | **Scope Assignment** | **Key Permissions** | **Required Actions** | -|-----------------|----------------------|---------------------|----------------------| -| [**Compute Gallery Artifacts Publisher**][compute-gallery-artifacts-publisher] (Built-in) | **Azure Compute Gallery** (the specific gallery resource, or its resource group) | *Gallery management:* Allows creating image definitions and image versions in that gallery. | `Microsoft.Compute/galleries/*` | -| [**Storage Account Contributor**][storage-contributor] (Built-in) | **Storage Account/Container** (the storage resource containing the VHD) | *Blob access:* Grants access to [list and read VHD file contents][storage]. | `Microsoft.Storage/storageAccounts/listKeys/action` | +| **Role (Type)** | **Scope Assignment** | **Key Permissions** | **Required Actions** | +|-------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------|-----------------------------------------------------| +| [**Compute Gallery Artifacts Publisher**][compute-gallery-artifacts-publisher] (Built-in) | **Azure Compute Gallery** (the specific gallery resource, or its resource group) | *Gallery management:* Allows creating image definitions and image versions in that gallery. | `Microsoft.Compute/galleries/*` | +| [**Storage Account Contributor**][storage-contributor] (Built-in) | **Storage Account/Container** (the storage resource containing the VHD) | *Blob access:* Grants access to [list and read VHD file contents][storage]. | `Microsoft.Storage/storageAccounts/listKeys/action` | !!! warning Each role should be assigned **only at the needed scope** (the specific gallery and storage resource). diff --git a/content/azure-cpi-errors.md b/content/azure-cpi-errors.md index b622fd7a4..0485ee4ee 100644 --- a/content/azure-cpi-errors.md +++ b/content/azure-cpi-errors.md @@ -1,6 +1,10 @@ +# Azure CPI Common Errors + ## Invalid Service Principal -> http_get_response - get_token - http error: 400 +```text +http_get_response - get_token - http error: 400 +``` Service principal is most likely invalid. Verify that client ID, client secret and tenant ID successfully work: @@ -14,71 +18,74 @@ If your service principal worked and you get the above error suddenly, it may be 2. Then choose your service principal, select `Configure` -- > `keys` -- > add a new key. - ## Exceeding quota limits of Core -> http_put - error: 409 message: { -> "error": { -> "code": "OperationNotAllowed", -> "message": "Operation results in exceeding quota limits of Core. Maximum allowed: 4, Current in use: 4, Additional requested: 1." -> } -> } +```json + http_put - error: 409 message: { + "error": { + "code": "OperationNotAllowed", + "message": "Operation results in exceeding quota limits of Core. Maximum allowed: 4, Current in use: 4, Additional requested: 1." + } + } +``` Either upgrade your trial account, or file a support ticket in the Azure portal to raise account quotas. - ## Network Interface In Use -> http_delete - error: 400 message: { -> "error": { -> "code": "NicInUse", -> "message": "Network Interface /.../networkInterfaces/dc0d3a9a-0b00-40d8-830d-41e6f4ac9809 is used by existing VM /.../virtualMachines/dc0d3a9a-0b00-40d8-830d-41e6f4ac9809.", -> "details": [] -> } -> } +```json + http_delete - error: 400 message: { + "error": { + "code": "NicInUse", + "message": "Network Interface /.../networkInterfaces/dc0d3a9a-0b00-40d8-830d-41e6f4ac9809 is used by existing VM /.../virtualMachines/dc0d3a9a-0b00-40d8-830d-41e6f4ac9809.", + "details": [] + } + } +``` This error indicates that unknown VM (to the Director) took up the IP that the Director is trying to assign to a new VM. Either let the Director know to not use this IP by including it in the reserved section of a subnet in your manual network, or make that IP available by terminating the unknown VM. - ## Limits of Premium Storage blob snapshots -> Error 100: Unknown CPI error 'Unknown' with message 'SnaphotOperationRateExceeded (409): The rate of snapshot blob calls is exceeded. +```text +Error 100: Unknown CPI error 'Unknown' with message 'SnaphotOperationRateExceeded (409): The rate of snapshot blob calls is exceeded. +``` The BOSH snapshot operation may be throttled if you do all of the following: - * Use Premium Storage for the Cloud Foundry VMs. +- Use Premium Storage for the Cloud Foundry VMs. - * Enable snapshot in `bosh.yml`. For more information on BOSH Snapshots, please go to https://bosh.io/docs/snapshots.html. +- Enable snapshot in `bosh.yml`. For more information on BOSH Snapshots, please go to . - ``` + ```yaml director: enable_snapshots: true ``` - * The time between consecutive snapshots by BOSH is less than **10 minutes**. The limits are documented in [Snapshots and Copy Blob for Premium Storage](https://azure.microsoft.com/en-us/documentation/articles/storage-premium-storage/#snapshots-and-copy-blob). +- The time between consecutive snapshots by BOSH is less than **10 minutes**. The limits are documented in [Snapshots and Copy Blob for Premium Storage](https://azure.microsoft.com/en-us/documentation/articles/storage-premium-storage/#snapshots-and-copy-blob). The workaround is: - * Disable snapshot temporarily. +- Disable snapshot temporarily. - ``` + ```yaml director: enable_snapshots: false ``` - * Adjust the snapshot interval to more than 10 minutes. - +* Adjust the snapshot interval to more than 10 minutes. ## Version mismatch between CPI and Stemcell -> Performing POST request: -> Post https://mbus-user:@10.0.0.4:6868/agent: dial tcp 10.0.0.4:6868: getsockopt: connection refused +```text +Performing POST request: + Post https://mbus-user:@10.0.0.4:6868/agent: dial tcp 10.0.0.4:6868: getsockopt: connection refused +``` For CPI v11 or later, the compatible stemcell version is v3181 or later. If the stemcell version is older than v3181, you may hit the following failure when deploying BOSH. It is recommended to use the latest version. For example, Stemcell v3232.5 or later, and CPI v12 or later. You may hit the issue [#135](https://github.com/cloudfoundry/bosh-azure-cpi-release/issues/135) if you still use an older stemcell than v3232.5. - ## Out of memory If you hit `Out of memory` or `Virtual memory exhausted`, please check whether you use Standard_A0 as instance_type. You should change instance_type to a VM size with more memory. diff --git a/content/azure-cpi.md b/content/azure-cpi.md index 6ac979651..2333cc670 100644 --- a/content/azure-cpi.md +++ b/content/azure-cpi.md @@ -1,8 +1,10 @@ +# Azure CPI Cloud Properties + This topic describes cloud properties for different resources created by the Azure CPI. ## AZs {: #azs } -* **availability_zone** [String, optional]: Availability zone to use for creating instances (available in v33+). Possible values: `'1'`, `'2'`, `'3'`. Read this [document](https://docs.microsoft.com/en-us/azure/availability-zones/az-overview) to get regions and VM sizes on Azure that support availability zones. [More details about availability zone](https://github.com/cloudfoundry/bosh-azure-cpi-release/tree/master/docs/advanced/availability-zone). +- **availability_zone** [String, optional]: Availability zone to use for creating instances (available in v33+). Possible values: `'1'`, `'2'`, `'3'`. Read this [document](https://docs.microsoft.com/en-us/azure/availability-zones/az-overview) to get regions and VM sizes on Azure that support availability zones. [More details about availability zone](https://github.com/cloudfoundry/bosh-azure-cpi-release/tree/master/docs/advanced/availability-zone). Example: @@ -14,21 +16,22 @@ azs: ``` --- + ## Networks {: #networks } ### Dynamic Network or Manual Network Schema for `cloud_properties` section: -* **resource\_group\_name** [String, optional]: Name of a resource group. If it is set, Azure CPI will search the virtual network and security group in this resource group. Otherwise, Azure CPI will search the virtual network and security group in `resource_group_name` in the global CPI settings. -* **virtual\_network\_name** [String, required]: Name of a virtual network. Example: `boshnet`. -* **subnet_name** [String, required]: Name of a subnet within virtual network. -* **security_group** [String, optional]: The [security group](https://azure.microsoft.com/en-us/documentation/articles/virtual-networks-nsg/) to apply to network interfaces of all VMs placed in this network. The security group of a network interface can be specified either in a VM type/extension (higher priority) or a network configuration (lower priority). If it's not specified in neither places, the default security group (specified by `default_security_group` in the global CPI settings) will be used. -* **application\_security\_groups** [Array, optional]: The [application security group](https://docs.microsoft.com/en-us/azure/virtual-network/security-overview#application-security-groups) to apply to network interfaces of all VMs placed in this network. The application security groups of a network interface can be specified either in a VM type/extension (higher priority) or a network configuration (lower priority). - * This property is supported in v31+. - * You must reference the [document](https://docs.microsoft.com/en-us/azure/virtual-network/create-network-security-group-preview) to register your subscription with this new feature. -* **ip_forwarding** [Boolean, optional]: The flag to enable [ip forwarding](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-network-network-interface#enable-or-disable-ip-forwarding) for network interfaces of all VMs placed in this network. The ip forwarding can be enabled/disabled either in a VM type/extension (higher priority) or a network configuration (lower priority). If it's not specified in neither places, the default value is `false`. Available in v35.3.0+. -* **accelerated_networking** [Boolean, optional]: The flag to enable [accelerated networking](https://docs.microsoft.com/en-us/azure/virtual-network/create-vm-accelerated-networking-cli) for network interfaces of all VMs placed in this network. The accelerated networking can be enabled/disabled either in a VM type/extension (higher priority) or a network configuration (lower priority). If it's not specified in neither places, the default value is `false`. Available in v35.4.0+. This feature needs ubuntu-xenial v81+ and [specific instance type](https://docs.microsoft.com/en-us/azure/virtual-network/create-vm-accelerated-networking-cli#supported-vm-instances). +- **resource\_group\_name** [String, optional]: Name of a resource group. If it is set, Azure CPI will search the virtual network and security group in this resource group. Otherwise, Azure CPI will search the virtual network and security group in `resource_group_name` in the global CPI settings. +- **virtual\_network\_name** [String, required]: Name of a virtual network. Example: `boshnet`. +- **subnet_name** [String, required]: Name of a subnet within virtual network. +- **security_group** [String, optional]: The [security group](https://azure.microsoft.com/en-us/documentation/articles/virtual-networks-nsg/) to apply to network interfaces of all VMs placed in this network. The security group of a network interface can be specified either in a VM type/extension (higher priority) or a network configuration (lower priority). If it's not specified in neither places, the default security group (specified by `default_security_group` in the global CPI settings) will be used. +- **application\_security\_groups** [Array, optional]: The [application security group](https://docs.microsoft.com/en-us/azure/virtual-network/security-overview#application-security-groups) to apply to network interfaces of all VMs placed in this network. The application security groups of a network interface can be specified either in a VM type/extension (higher priority) or a network configuration (lower priority). + - This property is supported in v31+. + - You must reference the [document](https://docs.microsoft.com/en-us/azure/virtual-network/create-network-security-group-preview) to register your subscription with this new feature. +- **ip_forwarding** [Boolean, optional]: The flag to enable [ip forwarding](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-network-network-interface#enable-or-disable-ip-forwarding) for network interfaces of all VMs placed in this network. The ip forwarding can be enabled/disabled either in a VM type/extension (higher priority) or a network configuration (lower priority). If it's not specified in neither places, the default value is `false`. Available in v35.3.0+. +- **accelerated_networking** [Boolean, optional]: The flag to enable [accelerated networking](https://docs.microsoft.com/en-us/azure/virtual-network/create-vm-accelerated-networking-cli) for network interfaces of all VMs placed in this network. The accelerated networking can be enabled/disabled either in a VM type/extension (higher priority) or a network configuration (lower priority). If it's not specified in neither places, the default value is `false`. Available in v35.4.0+. This feature needs ubuntu-xenial v81+ and [specific instance type](https://docs.microsoft.com/en-us/azure/virtual-network/create-vm-accelerated-networking-cli#supported-vm-instances). See [how to create a virtual network and subnets](azure-resources.md#virtual-network). @@ -55,7 +58,7 @@ networks: Schema for `cloud_properties` section: -* **resource\_group\_name** [String, optional]: Name of a resource group. If it is set, Azure CPI will search the public IP in this resource group. Otherwise, Azure CPI will search the public IP in `resource_group_name` in the global CPI settings. +- **resource\_group\_name** [String, optional]: Name of a resource group. If it is set, Azure CPI will search the public IP in this resource group. Otherwise, Azure CPI will search the public IP in `resource_group_name` in the global CPI settings. See [how to create public IP](azure-resources.md#public-ips) to use with vip networks. @@ -70,126 +73,147 @@ networks: ``` --- + ## VM Types / VM Extensions {: #resource-pools } Schema for `cloud_properties` section: -* **instance_type** [String, required]: Type of the [instance](https://azure.microsoft.com/en-us/documentation/articles/virtual-machines-linux-sizes/). Example: `Standard_A2`. [Basic Tier Virtual Machines](https://azure.microsoft.com/en-us/blog/basic-tier-virtual-machines-2/) should not be used if you need to bind the instance to Azure Load Balancer (ALB), because Basic Tier VM doesn't support ALB. -* **root_disk** [Hash, optional]: OS disk of custom size. - * **size** [Integer, optional]: Specifies the disk size in MiB. - * The size must be greater than 3 * 1024 and less than the max disk size for [unmanaged](https://azure.microsoft.com/en-us/pricing/details/storage/unmanaged-disks/) or [managed](https://azure.microsoft.com/en-us/pricing/details/managed-disks/) disk. Please always use `N * 1024` as the size because Azure always uses GiB but not MiB. - * It has a default value `30 * 1024` only when ephemeral_disk.use\_root\_disk is set to true. - * **disk_encryption_set_name** [String, optional]: The Azure Disk Encryption Set name to use when creating the root disk. This is used to encrypt the disk with customer provided keys rather than the default Azure provided encryption keys. Available since v52.0.0. -* **caching** [String, optional]: Type of the disk caching of the VMs' OS disks. It can be either `None`, `ReadOnly` or `ReadWrite`. Default is `ReadWrite`. -* **ephemeral_disk** [Hash, optional]: Ephemeral disk to apply for all VMs that are in this VM type/extension. By default a data disk with the default size as below will be created as the ephemeral disk. - * **use\_root\_disk** [Boolean, optional]: Enable to use OS disk to store the ephemeral data. The default value is false. When it is true, ephemeral_disk.size will not be used. - * **size** [Integer, optional]: Specifies the disk size in MiB. If this is not set, the default size as below will be used. The size of the ephemeral disk for the BOSH VM should be larger than or equal to `30*1024` MiB. Please always use `N * 1024` as the size because Azure always uses GiB not MiB. - * If the Azure temporary disk size for the instance type is less than `30*1024` MiB, the default size is `30*1024` MiB because the space may not be enough. - * If the Azure temporary disk size for the instance type is larger than `1000*1024` MiB, the default size is `1000*1024` MiB because it is not expected to use such a large ephemeral disk in CF currently. - * Otherwise, the Azure temporary disk size will be used as the default size. See more information about [Azure temporary disk size](https://azure.microsoft.com/en-us/documentation/articles/virtual-machines-linux-sizes/). - * **caching** [String, optional]: Type of the disk caching. It can be either `None`, `ReadOnly` or `ReadWrite`. Default is `None`. - * **type** [String, optional]: Storage account type. Valid only when `use_managed_disks` is `true`. It can be either `Standard_LRS`, `Premium_LRS` or `PremiumV2_LRS`. You can click [**HERE**](http://azure.microsoft.com/en-us/pricing/details/storage/) to learn more about the type of Azure storage account. For `PremiumV2_LRS`, you have to set caching to `None` since `PremiumV2_LRS` does currently not support caching. - * **iops** [Integer, optional]: IOPS of the disk. If you need more IOPS than the baseline offers, you can increase the IOPS of the disks. For more details, see [Premium SSD v2 performance](https://learn.microsoft.com/azure/virtual-machines/disks-types#premium-ssd-v2-performance). Only supported for `PremiumV2_LRS` - * **mbps** [Integer, optional]: Throughput in MB/s of the disk. If you need more throughput than the baseline offers, you can increase the throughput of the disks. For more details, see [Premium SSD v2 performance](https://learn.microsoft.com/azure/virtual-machines/disks-types#premium-ssd-v2-performance). Only supported for `PremiumV2_LRS` - * **disk_encryption_set_name** [String, optional]: The Azure Disk Encryption Set name to use when creating the ephemeral disk. This is used to encrypt the disk with customer provided keys rather than the default Azure provided encryption keys. Available since v52.0.0. - -* **load_balancer** [String, optional]: Name of a [load balancer](https://azure.microsoft.com/en-us/documentation/articles/load-balancer-overview/) the VMs should belong to. - * _Notes:_ - * You need to create the load balancer manually before configuring it. - * [Basic Tier Virtual Machines](https://azure.microsoft.com/en-us/blog/basic-tier-virtual-machines-2/) (Example: `Basic_A1`) doesn't support Azure Load Balancer. - * If `availability_zone` is specified for the VM, [standard sku load balancer](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-standard-overview) must be used, as `basic sku load balancer` does not work for zone. - * In CPI v37.6.0+, you can configure multiple Load Balancers (using a comma-delimited string). - * This property is equivalent to the `load_balancer/name` property below. -* **load_balancer** [Array or Hash, optional]: The [load balancers](https://azure.microsoft.com/en-us/documentation/articles/load-balancer-overview/) the VMs should belong to. - * _Notes:_ - * This property is supported in CPI v35.5.0+. In earlier versions, use the String property above instead. - * In CPI [v37.7.0+](https://github.com/cloudfoundry/bosh-azure-cpi-release/releases/tag/v37.7.0), you can configure multiple Load Balancers, using an Array of Hashes with the properties below. - * In CPI [v35.5.0+](https://github.com/cloudfoundry/bosh-azure-cpi-release/releases/tag/v35.5.0), you can configure a single Load Balancer, using a single Hash with the properties below. - * You need to create the load balancer(s) manually before configuring them. - * [Basic Tier Virtual Machines](https://azure.microsoft.com/en-us/blog/basic-tier-virtual-machines-2/) (Example: `Basic_A1`) doesn't support Azure Load Balancer. - * If `availability_zone` is specified for the VM, [standard sku load balancer](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-standard-overview) must be used, as `basic sku load balancer` does not work for zone. - * **name** [String, required]: The name of the load balancer. - * **resource\_group\_name** [String, optional]: The name of the load balancer's resource group. Default value is the `resource_group_name` specified in the global CPI settings. - * **backend\_pool\_name** [String, optional]: The name of the load balancer backend address pool which VMs' IPs should be attached to. If not specified, defaults to the load balancer's "first" backend pool (as returned by the Azure API). - * This property is supported in CPI [v37.7.0+](https://github.com/cloudfoundry/bosh-azure-cpi-release/releases/tag/v37.7.0). -* **application_gateway** [String, optional]: Name of the [application gateway](https://azure.microsoft.com/en-us/services/application-gateway/) which the VMs should be attached to. - * _Notes:_ - * This property is supported in CPI v28+. - * You need to create the application gateway manually before configuring it. Please refer to [the guidance](https://github.com/cloudfoundry/bosh-azure-cpi-release/tree/master/docs/advanced/application-gateway). - * This property is equivalent to the `application_gateway/name` property below. -* **application_gateway** [Array or Hash, optional]: The [application gateways](https://azure.microsoft.com/en-us/services/application-gateway/) the VMs should be attached to. - * _Notes:_ - * This property is supported in CPI [v37.7.0+](https://github.com/cloudfoundry/bosh-azure-cpi-release/releases/tag/v37.7.0). In earlier versions, use the String property above instead. - * You need to create the application gateway(s) manually before configuring them. Please refer to [the guidance](https://github.com/cloudfoundry/bosh-azure-cpi-release/tree/master/docs/advanced/application-gateway). - * **name** [String, required]: The name of the application gateway. - * **resource\_group\_name** [String, optional]: The name of the application gateway's resource group. Default value is the `resource_group_name` specified in the global CPI settings. - * **backend\_pool\_name** [String, optional]: The name of the application gateway backend address pool which VMs' IPs should be attached to. If not specified, defaults to the application gateway's "first" backend pool (as returned by the Azure API). -* **security_group** [String, optional]: The [security group](https://azure.microsoft.com/en-us/documentation/articles/virtual-networks-nsg/) to apply to network interfaces of all VMs who have this VM type/extension. The security group of a network interface can be specified either in a VM type/extension (higher priority) or a network configuration (lower priority). If it's not specified in neither places, the default security group (specified by `default_security_group` in the global CPI settings) will be used. -* **application\_security\_groups** [Array, optional]: The [application security group](https://docs.microsoft.com/en-us/azure/virtual-network/security-overview#application-security-groups) to apply to network interfaces of all VMs who have this VM type/extension. The application security groups of a network interface can be specified either in a VM type/extension (higher priority) or a network configuration (lower priority). - * This property is supported in v31+. - * You must reference the [document](https://docs.microsoft.com/en-us/azure/virtual-network/create-network-security-group-preview) to register your subscription with this new feature. -* **ip_forwarding** [Boolean, optional]: The flag to enable [ip forwarding](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-network-network-interface#enable-or-disable-ip-forwarding) for network interfaces of all VMs who have this VM type/extension. The ip forwarding can be enabled/disabled either in a VM type/extension (higher priority) or a network configuration (lower priority). If it's not specified in neither places, the default value is `false`. Available in v35.3.0+. -* **accelerated_networking** [Boolean, optional]: The flag to enable [accelerated networking](https://docs.microsoft.com/en-us/azure/virtual-network/create-vm-accelerated-networking-cli) for network interfaces of all VMs who have this VM type/extension. The accelerated networking can be enabled/disabled either in a VM type/extension (higher priority) or a network configuration (lower priority). If it's not specified in neither places, the default value is `false`. Available in v35.4.0+. This feature needs ubuntu-xenial v81+ and [specific instance type](https://docs.microsoft.com/en-us/azure/virtual-network/create-vm-accelerated-networking-cli#supported-vm-instances). - -* **assign\_dynamic\_public\_ip** [Boolean, optional]: Enable to create and assign dynamic public IP to the VM automatically (to solve the [azure SNAT issue](https://github.com/cloudfoundry/bosh-azure-cpi-release/issues/217)). Default value is `false`. Only the VM without vip will be assigned a dynamic public IP when this value is set to true, and the dynamic public IP will be deleted when the VM is deleted. - -* **availability_zone** [String, optional]: Availability zone to use for creating instances (available in v33+). Possible values: `'1'`, `'2'`, `'3'`. Read this [document](https://docs.microsoft.com/en-us/azure/availability-zones/az-overview) to get regions and VM sizes on Azure that support availability zones. [More details about availability zone](https://github.com/cloudfoundry/bosh-azure-cpi-release/tree/master/docs/advanced/availability-zone). -* **availability_set** [String, optional]: Name of an [availability set](https://azure.microsoft.com/en-us/documentation/articles/virtual-machines-manage-availability/) to use for VMs. [More details](https://github.com/cloudfoundry/bosh-azure-cpi-release/tree/master/docs/advanced/deploy-cloudfoundry-for-enterprise#availability-set). - * If available set does not exist, it will be automatically created. - * If `availability_set` is not specified, Azure CPI will search `env.bosh.group` as the name of availability set. +- **instance_type** [String, required]: Type of the [instance](https://azure.microsoft.com/en-us/documentation/articles/virtual-machines-linux-sizes/). Example: `Standard_A2`. [Basic Tier Virtual Machines](https://azure.microsoft.com/en-us/blog/basic-tier-virtual-machines-2/) should not be used if you need to bind the instance to Azure Load Balancer (ALB), because Basic Tier VM doesn't support ALB. + +- **root_disk** [Hash, optional]: OS disk of custom size. + - **size** [Integer, optional]: Specifies the disk size in MiB. + - The size must be greater than 3 * 1024 and less than the max disk size for [unmanaged](https://azure.microsoft.com/en-us/pricing/details/storage/unmanaged-disks/) or [managed](https://azure.microsoft.com/en-us/pricing/details/managed-disks/) disk. Please always use `N * 1024` as the size because Azure always uses GiB but not MiB. + - It has a default value `30 * 1024` only when ephemeral_disk.use\_root\_disk is set to true. + - **disk_encryption_set_name** [String, optional]: The Azure Disk Encryption Set name to use when creating the root disk. This is used to encrypt the disk with customer provided keys rather than the default Azure provided encryption keys. Available since v52.0.0. +- **caching** [String, optional]: Type of the disk caching of the VMs' OS disks. It can be either `None`, `ReadOnly` or `ReadWrite`. Default is `ReadWrite`. + +- **ephemeral_disk** [Hash, optional]: Ephemeral disk to apply for all VMs that are in this VM type/extension. By default a data disk with the default size as below will be created as the ephemeral disk. + - **use\_root\_disk** [Boolean, optional]: Enable to use OS disk to store the ephemeral data. The default value is false. When it is true, ephemeral_disk.size will not be used. + - **size** [Integer, optional]: Specifies the disk size in MiB. If this is not set, the default size as below will be used. The size of the ephemeral disk for the BOSH VM should be larger than or equal to `30*1024` MiB. Please always use `N * 1024` as the size because Azure always uses GiB not MiB. + - If the Azure temporary disk size for the instance type is less than `30*1024` MiB, the default size is `30*1024` MiB because the space may not be enough. + - If the Azure temporary disk size for the instance type is larger than `1000*1024` MiB, the default size is `1000*1024` MiB because it is not expected to use such a large ephemeral disk in CF currently. + - Otherwise, the Azure temporary disk size will be used as the default size. See more information about [Azure temporary disk size](https://azure.microsoft.com/en-us/documentation/articles/virtual-machines-linux-sizes/). + - **caching** [String, optional]: Type of the disk caching. It can be either `None`, `ReadOnly` or `ReadWrite`. Default is `None`. + - **type** [String, optional]: Storage account type. Valid only when `use_managed_disks` is `true`. It can be either `Standard_LRS`, `Premium_LRS` or `PremiumV2_LRS`. You can click [**HERE**](http://azure.microsoft.com/en-us/pricing/details/storage/) to learn more about the type of Azure storage account. For `PremiumV2_LRS`, you have to set caching to `None` since `PremiumV2_LRS` does currently not support caching. + - **iops** [Integer, optional]: IOPS of the disk. If you need more IOPS than the baseline offers, you can increase the IOPS of the disks. For more details, see [Premium SSD v2 performance](https://learn.microsoft.com/azure/virtual-machines/disks-types#premium-ssd-v2-performance). Only supported for `PremiumV2_LRS` + - **mbps** [Integer, optional]: Throughput in MB/s of the disk. If you need more throughput than the baseline offers, you can increase the throughput of the disks. For more details, see [Premium SSD v2 performance](https://learn.microsoft.com/azure/virtual-machines/disks-types#premium-ssd-v2-performance). Only supported for `PremiumV2_LRS` + - **disk_encryption_set_name** [String, optional]: The Azure Disk Encryption Set name to use when creating the ephemeral disk. This is used to encrypt the disk with customer provided keys rather than the default Azure provided encryption keys. Available since v52.0.0. + +- **load_balancer** [String, optional]: Name of a [load balancer](https://azure.microsoft.com/en-us/documentation/articles/load-balancer-overview/) the VMs should belong to. + !!! note + - You need to create the load balancer manually before configuring it. + - [Basic Tier Virtual Machines](https://azure.microsoft.com/en-us/blog/basic-tier-virtual-machines-2/) (Example: `Basic_A1`) doesn't support Azure Load Balancer. + - If `availability_zone` is specified for the VM, [standard sku load balancer](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-standard-overview) must be used, as `basic sku load balancer` does not work for zone. + + - In CPI v37.6.0+, you can configure multiple Load Balancers (using a comma-delimited string). + - This property is equivalent to the `load_balancer/name` property below. + +- **load_balancer** [Array or Hash, optional]: The [load balancers](https://azure.microsoft.com/en-us/documentation/articles/load-balancer-overview/) the VMs should belong to. + + !!! note + - This property is supported in CPI v35.5.0+. In earlier versions, use the String property above instead. + - In CPI [v37.7.0+](https://github.com/cloudfoundry/bosh-azure-cpi-release/releases/tag/v37.7.0), you can configure multiple Load Balancers, using an Array of Hashes with the properties below. + - In CPI [v35.5.0+](https://github.com/cloudfoundry/bosh-azure-cpi-release/releases/tag/v35.5.0), you can configure a single Load Balancer, using a single Hash with the properties below. + - You need to create the load balancer(s) manually before configuring them. + - [Basic Tier Virtual Machines](https://azure.microsoft.com/en-us/blog/basic-tier-virtual-machines-2/) (Example: `Basic_A1`) doesn't support Azure Load Balancer. + - If `availability_zone` is specified for the VM, [standard sku load balancer](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-standard-overview) must be used, as `basic sku load balancer` does not work for zone. + + - **name** [String, required]: The name of the load balancer. + - **resource\_group\_name** [String, optional]: The name of the load balancer's resource group. Default value is the `resource_group_name` specified in the global CPI settings. + - **backend\_pool\_name** [String, optional]: The name of the load balancer backend address pool which VMs' IPs should be attached to. If not specified, defaults to the load balancer's "first" backend pool (as returned by the Azure API). + - This property is supported in CPI [v37.7.0+](https://github.com/cloudfoundry/bosh-azure-cpi-release/releases/tag/v37.7.0). + +- **application_gateway** [String, optional]: Name of the [application gateway](https://azure.microsoft.com/en-us/services/application-gateway/) which the VMs should be attached to. + + !!! note + - This property is supported in CPI v28+. + - You need to create the application gateway manually before configuring it. Please refer to [the guidance](https://github.com/cloudfoundry/bosh-azure-cpi-release/tree/master/docs/advanced/application-gateway). + - This property is equivalent to the `application_gateway/name` property below. + +- **application_gateway** [Array or Hash, optional]: The [application gateways](https://azure.microsoft.com/en-us/services/application-gateway/) the VMs should be attached to. + + !!! note + - This property is supported in CPI [v37.7.0+](https://github.com/cloudfoundry/bosh-azure-cpi-release/releases/tag/v37.7.0). In earlier versions, use the String property above instead. + - You need to create the application gateway(s) manually before configuring them. Please refer to [the guidance](https://github.com/cloudfoundry/bosh-azure-cpi-release/tree/master/docs/advanced/application-gateway). + + - **name** [String, required]: The name of the application gateway. + - **resource\_group\_name** [String, optional]: The name of the application gateway's resource group. Default value is the `resource_group_name` specified in the global CPI settings. + - **backend\_pool\_name** [String, optional]: The name of the application gateway backend address pool which VMs' IPs should be attached to. If not specified, defaults to the application gateway's "first" backend pool (as returned by the Azure API). + +- **security_group** [String, optional]: The [security group](https://azure.microsoft.com/en-us/documentation/articles/virtual-networks-nsg/) to apply to network interfaces of all VMs who have this VM type/extension. The security group of a network interface can be specified either in a VM type/extension (higher priority) or a network configuration (lower priority). If it's not specified in neither places, the default security group (specified by `default_security_group` in the global CPI settings) will be used. + +- **application\_security\_groups** [Array, optional]: The [application security group](https://docs.microsoft.com/en-us/azure/virtual-network/security-overview#application-security-groups) to apply to network interfaces of all VMs who have this VM type/extension. The application security groups of a network interface can be specified either in a VM type/extension (higher priority) or a network configuration (lower priority). + - This property is supported in v31+. + - You must reference the [document](https://docs.microsoft.com/en-us/azure/virtual-network/create-network-security-group-preview) to register your subscription with this new feature. + +- **ip_forwarding** [Boolean, optional]: The flag to enable [ip forwarding](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-network-network-interface#enable-or-disable-ip-forwarding) for network interfaces of all VMs who have this VM type/extension. The ip forwarding can be enabled/disabled either in a VM type/extension (higher priority) or a network configuration (lower priority). If it's not specified in neither places, the default value is `false`. Available in v35.3.0+. + +- **accelerated_networking** [Boolean, optional]: The flag to enable [accelerated networking](https://docs.microsoft.com/en-us/azure/virtual-network/create-vm-accelerated-networking-cli) for network interfaces of all VMs who have this VM type/extension. The accelerated networking can be enabled/disabled either in a VM type/extension (higher priority) or a network configuration (lower priority). If it's not specified in neither places, the default value is `false`. Available in v35.4.0+. This feature needs ubuntu-xenial v81+ and [specific instance type](https://docs.microsoft.com/en-us/azure/virtual-network/create-vm-accelerated-networking-cli#supported-vm-instances). + +- **assign\_dynamic\_public\_ip** [Boolean, optional]: Enable to create and assign dynamic public IP to the VM automatically (to solve the [azure SNAT issue](https://github.com/cloudfoundry/bosh-azure-cpi-release/issues/217)). Default value is `false`. Only the VM without vip will be assigned a dynamic public IP when this value is set to true, and the dynamic public IP will be deleted when the VM is deleted. + + +- **availability_zone** [String, optional]: Availability zone to use for creating instances (available in v33+). Possible values: `'1'`, `'2'`, `'3'`. Read this [document](https://docs.microsoft.com/en-us/azure/availability-zones/az-overview) to get regions and VM sizes on Azure that support availability zones. [More details about availability zone](https://github.com/cloudfoundry/bosh-azure-cpi-release/tree/master/docs/advanced/availability-zone). + +- **availability_set** [String, optional]: Name of an [availability set](https://azure.microsoft.com/en-us/documentation/articles/virtual-machines-manage-availability/) to use for VMs. [More details](https://github.com/cloudfoundry/bosh-azure-cpi-release/tree/master/docs/advanced/deploy-cloudfoundry-for-enterprise#availability-set). + - If available set does not exist, it will be automatically created. + - If `availability_set` is not specified, Azure CPI will search `env.bosh.group` as the name of availability set. 1. [bosh release](https://bosh.io/releases/github.com/cloudfoundry/bosh?all=1) v258+ will generate a value for `env.bosh.group` automatically. 1. On Azure the length of the availability set name must be between 1 and 80 characters. The name got from `env.bosh.group` may be too long. CPI will truncate the name to the following format `az-MD5-[LAST-40-CHARACTERS-OF-GROUP]` if the length is greater than 80. - * CPI v27+ will delete the empty availability set. - * Only one of `availability_zone` and `availability_set` is allowed to be configured for a VM. If `availability_zone` is specified, the VM will be in a zone and not in any availability set. -* **platform\_update\_domain_count** [Integer, optional]: The count of [update domain](https://azure.microsoft.com/en-us/documentation/articles/virtual-machines-manage-availability/) in the availability set. - * For Azure, the default value is `5`. - * For Azure Stack, the default value is `1`. -* **platform\_fault\_domain_count** [Integer, optional]: The count of [fault domain](https://azure.microsoft.com/en-us/documentation/articles/virtual-machines-manage-availability/) in the availability set. - * For Azure, the default value of an unmanaged availability set is `3`. The default value of a managed availability set is `2`, because [some regions don't support 3 fault domains for now](https://docs.microsoft.com/en-us/azure/virtual-machines/virtual-machines-windows-manage-availability#configure-multiple-virtual-machines-in-an-availability-set-for-redundancy) - * For Azure Stack, the default value is `1`. Before 1802 update, only `1` is allowed. After [1802 update](https://docs.microsoft.com/en-us/azure/azure-stack/azure-stack-update-1802), you can configure up to 3 fault domains. - -* **storage\_account\_name** [String, optional]: Storage account for VMs. Valid only when `use_managed_disks` is `false`. If this is not set, the VMs will be created in the default storage account. See [this document](https://github.com/cloudfoundry/bosh-azure-cpi-release/blob/master/docs/advanced/deploy-cloudfoundry-with-multiple-storage-accounts/README.md) for more details on why this option exists. - * If you use `DS-series` or `GS-series` as `instance_type`, you should set this to a premium storage account. See more information about [Azure premium storage](https://azure.microsoft.com/en-us/documentation/articles/storage-premium-storage-preview-portal/). See [available regions](http://azure.microsoft.com/en-us/regions/#services) where you can create premium storage accounts. - * If you use a different storage account which must be in the same resource group, please make sure: + - CPI v27+ will delete the empty availability set. + - Only one of `availability_zone` and `availability_set` is allowed to be configured for a VM. If `availability_zone` is specified, the VM will be in a zone and not in any availability set. +- **platform\_update\_domain_count** [Integer, optional]: The count of [update domain](https://azure.microsoft.com/en-us/documentation/articles/virtual-machines-manage-availability/) in the availability set. + - For Azure, the default value is `5`. + - For Azure Stack, the default value is `1`. +- **platform\_fault\_domain_count** [Integer, optional]: The count of [fault domain](https://azure.microsoft.com/en-us/documentation/articles/virtual-machines-manage-availability/) in the availability set. + - For Azure, the default value of an unmanaged availability set is `3`. The default value of a managed availability set is `2`, because [some regions don't support 3 fault domains for now](https://docs.microsoft.com/en-us/azure/virtual-machines/virtual-machines-windows-manage-availability#configure-multiple-virtual-machines-in-an-availability-set-for-redundancy) + - For Azure Stack, the default value is `1`. Before 1802 update, only `1` is allowed. After [1802 update](https://docs.microsoft.com/en-us/azure/azure-stack/azure-stack-update-1802), you can configure up to 3 fault domains. + +- **storage\_account\_name** [String, optional]: Storage account for VMs. Valid only when `use_managed_disks` is `false`. If this is not set, the VMs will be created in the default storage account. See [this document](https://github.com/cloudfoundry/bosh-azure-cpi-release/blob/master/docs/advanced/deploy-cloudfoundry-with-multiple-storage-accounts/README.md) for more details on why this option exists. + - If you use `DS-series` or `GS-series` as `instance_type`, you should set this to a premium storage account. See more information about [Azure premium storage](https://azure.microsoft.com/en-us/documentation/articles/storage-premium-storage-preview-portal/). See [available regions](http://azure.microsoft.com/en-us/regions/#services) where you can create premium storage accounts. + - If you use a different storage account which must be in the same resource group, please make sure: 1. The permissions for the container `stemcell` in the default storage account is set to `Public read access for blobs only`. 1. A table `stemcells` is created in the default storage account. 1. Two containers `bosh` and `stemcell` are created in the new storage account. - * If this storage account does not exist, it can be created automatically by Azure CPI. But you must specify storage\_account\_type and make sure: + - If this storage account does not exist, it can be created automatically by Azure CPI. But you must specify storage\_account\_type and make sure: 1. The name must be **unique within Azure**. 1. **The name must be between 3 and 24 characters in length and use numbers and lower-case letters only**. - * If you use a pattern `*keyword*`. CPI will filter all storage accounts under the default resource group by the pattern and pick one available storage account to create the VM. + - If you use a pattern `*keyword*`. CPI will filter all storage accounts under the default resource group by the pattern and pick one available storage account to create the VM. 1. The pattern must start with `*` and end with `*`. 1. The keyword must only contain numbers and lower-case letters because of the naming rule of storage account name. 1. The rule to select an available storage account is to check the number of disks under the container `bosh` does not exceed the limitation. 1. The default number of disks limitation is 30 but you can specify it in **storage\_account\_max\_disk\_number**. -* **storage\_account\_type** [String, optional]: Storage account type. You can click [**HERE**](http://azure.microsoft.com/en-us/pricing/details/storage/) to learn more about the type of Azure storage account. - * When `use_managed_disks` is `true`, the root disk's type is specified by this property. It can be either `Standard_LRS` or `Premium_LRS`. If not specified, `Premium_LRS` will be used when its `instance_type` supports premium storage, otherwise, `Standard_LRS` will be used. - * When `use_managed_disks` is `false`, the newly-created storage account's type is specified by this property. It can be either `Standard_LRS`, `Standard_ZRS`, `Standard_GRS`, `Standard_RAGRS` or `Premium_LRS`. It's required if the storage account does not exist. -* **storage\_account\_max\_disk\_number** [Integer, optional]: Number of disks limitation in a storage account. Valid only when `use_managed_disks` is `false`. Default value is 30. This will be used only when **storage\_account\_name** is a pattern. - * Every storage account has a limitation to host disks. You may hit the performance issue if you create too many disks in one storage account. - * The maximum number of disks of a standard storage account is 40 because the maximum IOPS of a standard storage account is 20,000 and the maximum IOPS of a standard disk is 500. - * If you are using premium storage account, Azure maps the disk size (rounded up) to the nearest Premium Storage Disk option (P10, P20 and P30). For example, a disk of size 100 GiB is classified as a P10 option. + +- **storage\_account\_type** [String, optional]: Storage account type. You can click [**HERE**](http://azure.microsoft.com/en-us/pricing/details/storage/) to learn more about the type of Azure storage account. + - When `use_managed_disks` is `true`, the root disk's type is specified by this property. It can be either `Standard_LRS` or `Premium_LRS`. If not specified, `Premium_LRS` will be used when its `instance_type` supports premium storage, otherwise, `Standard_LRS` will be used. + - When `use_managed_disks` is `false`, the newly-created storage account's type is specified by this property. It can be either `Standard_LRS`, `Standard_ZRS`, `Standard_GRS`, `Standard_RAGRS` or `Premium_LRS`. It's required if the storage account does not exist. + +- **storage\_account\_max\_disk\_number** [Integer, optional]: Number of disks limitation in a storage account. Valid only when `use_managed_disks` is `false`. Default value is 30. This will be used only when **storage\_account\_name** is a pattern. + - Every storage account has a limitation to host disks. You may hit the performance issue if you create too many disks in one storage account. + - The maximum number of disks of a standard storage account is 40 because the maximum IOPS of a standard storage account is 20,000 and the maximum IOPS of a standard disk is 500. + - If you are using premium storage account, Azure maps the disk size (rounded up) to the nearest Premium Storage Disk option (P10, P20 and P30). For example, a disk of size 100 GiB is classified as a P10 option. 1. The maximum number of disks of a premium storage account is 280 if you are using P10 (128 GiB) as your disk type. 1. The maximum number of disks of a premium storage account is 70 if you are using P20 (512 GiB) as your disk type. 1. The maximum number of disks of a premium storage account is 35 if you are using P30 (1024 GiB) as your disk type. - * **storage\_account\_max\_disk\_number** should be less than the maximum number. Suggest you to use (MAX - 10) as the value because CPI always creates VMs in parallel. - * Please see more information about [azure-subscription-service-limits](https://azure.microsoft.com/en-us/documentation/articles/azure-subscription-service-limits/). -* **storage\_account\_location** [String, optional]: Location of the storage account. This configuration is deprecated in CPI v25+: if you specify a storage account which does not exist, CPI (v25+) will create it automatically in the same location as VMs' VNET. + - **storage\_account\_max\_disk\_number** should be less than the maximum number. Suggest you to use (MAX - 10) as the value because CPI always creates VMs in parallel. + - Please see more information about [azure-subscription-service-limits](https://azure.microsoft.com/en-us/documentation/articles/azure-subscription-service-limits/). -* **resource\_group\_name** [String, optional]: Name of a resource group (Available in v26+). If it is set, related resources will be created in this resource group; otherwise, they will be created in `resource_group_name` specified in the global CPI settings. The resources affected by this property are: +- **storage\_account\_location** [String, optional]: Location of the storage account. This configuration is deprecated in CPI v25+: if you specify a storage account which does not exist, CPI (v25+) will create it automatically in the same location as VMs' VNET. + +- **resource\_group\_name** [String, optional]: Name of a resource group (Available in v26+). If it is set, related resources will be created in this resource group; otherwise, they will be created in `resource_group_name` specified in the global CPI settings. The resources affected by this property are: 1. Virtual Machine 1. Network Interface Card 1. Managed Disks (including OS disk, ephemeral disk, persistent disk and snapshot) 1. Dynamic Public IP for the VM 1. Availability Set -* **managed_identity** [Hash, optional]: [Azure Managed Identity](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) to assign to the VM. With a Managed Identity, the VM can access Azure resources (like Storage Accounts and KeyVaults) without needing to store credentials on the VM. - * **type** [String, optional]: You can choose between UserAssigned and SystemAssigned. The default value is SystemAssigned. - * **user_assigned_identity_name** [String, required]: Specifies the name of the Managed Identity. +- **managed_identity** [Hash, optional]: [Azure Managed Identity](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) to assign to the VM. With a Managed Identity, the VM can access Azure resources (like Storage Accounts and KeyVaults) without needing to store credentials on the VM. + - **type** [String, optional]: You can choose between UserAssigned and SystemAssigned. The default value is SystemAssigned. + - **user_assigned_identity_name** [String, required]: Specifies the name of the Managed Identity. -* **tags** [Hash, optional]: Custom tags of VMs (Available in v35.4.0+). They are name-value pairs that are used to organize VMs. -* **capacity_reservation_group** [String, optional]: The name of the [capacity reservation](https://learn.microsoft.com/en-us/azure/virtual-machines/capacity-reservation-overview) group to use for the VM. Available since v53.0.0+. The value must be a valid capacity reservation group name in the same resource group as the VM. -* **capacity_reservation_group_id** [String, optional]: The ID of the [capacity reservation](https://learn.microsoft.com/en-us/azure/virtual-machines/capacity-reservation-overview) group to use for the VM. Available since v53.0.0+. The value must be a valid Azure Resource ID for a capacity reservation group that the configured SPN can access. +- **tags** [Hash, optional]: Custom tags of VMs (Available in v35.4.0+). They are name-value pairs that are used to organize VMs. +- **capacity_reservation_group** [String, optional]: The name of the [capacity reservation](https://learn.microsoft.com/en-us/azure/virtual-machines/capacity-reservation-overview) group to use for the VM. Available since v53.0.0+. The value must be a valid capacity reservation group name in the same resource group as the VM. +- **capacity_reservation_group_id** [String, optional]: The ID of the [capacity reservation](https://learn.microsoft.com/en-us/azure/virtual-machines/capacity-reservation-overview) group to use for the VM. Available since v53.0.0+. The value must be a valid Azure Resource ID for a capacity reservation group that the configured SPN can access. The capacity reservation group can be used when the resource is not part of the resource_group. The capacity of the reservation group can be shared over different subscriptions in the same region. [ResourcesSharingProfile](https://learn.microsoft.com/en-us/rest/api/compute/capacity-reservation-groups/list-by-subscription?view=rest-compute-2025-04-01&tabs=HTTP#resourcesharingprofile) @@ -309,7 +333,6 @@ vm_extensions: user_assigned_identity_name: ``` - The above `vm_extensions` cloud configuration examples are referenced within the deployment manifest as such: ```yaml @@ -330,18 +353,19 @@ instance_groups: ``` --- + ## Disk Types {: #disk-pools } -* **name** [String, required]: Name of the disk type. -* **disk_size** [Integer, required]: Size of the disk in MiB. On Azure the disk size must be greater than 1 * 1024 and less than the max disk size for [unmanaged](https://azure.microsoft.com/en-us/pricing/details/storage/unmanaged-disks/) or [managed](https://azure.microsoft.com/en-us/pricing/details/managed-disks/) disk. Please always use `N * 1024` as the size because Azure always uses GiB not MiB. +- **name** [String, required]: Name of the disk type. +- **disk_size** [Integer, required]: Size of the disk in MiB. On Azure the disk size must be greater than 1 * 1024 and less than the max disk size for [unmanaged](https://azure.microsoft.com/en-us/pricing/details/storage/unmanaged-disks/) or [managed](https://azure.microsoft.com/en-us/pricing/details/managed-disks/) disk. Please always use `N * 1024` as the size because Azure always uses GiB not MiB. Schema for `cloud_properties` section: -* **caching** [String, optional]: Type of the disk caching. It can be either `None`, `ReadOnly` or `ReadWrite`. Default is `None`. -* **storage\_account\_type** [String, optional]: Storage account type. Valid only when `use_managed_disks` is `true`. It can be either `Standard_LRS`, `Premium_LRS` or `PremiumV2_LRS`. You can click [**HERE**](http://azure.microsoft.com/en-us/pricing/details/storage/) to learn more about the type of Azure storage account. For `PremiumV2_LRS`, you have to set caching to `None` since `PremiumV2_LRS` does currently not support caching. -* **iops** [Integer, optional]: IOPS of the disk. If you need more IOPS than the baseline offers, you can increase the IOPS of the disks. For more details, see [Premium SSD v2 performance](https://learn.microsoft.com/azure/virtual-machines/disks-types#premium-ssd-v2-performance). Only supported for `PremiumV2_LRS` -* **mbps** [Integer, optional]: Throughput in MB/s of the disk. If you need more throughput than the baseline offers, you can increase the throughput of the disks. For more details, see [Premium SSD v2 performance](https://learn.microsoft.com/azure/virtual-machines/disks-types#premium-ssd-v2-performance). Only supported for `PremiumV2_LRS` -* **disk_encryption_set_name** [String, optional]: The Azure Disk Encryption Set name to use when creating the persistent disk. This is used to encrypt the disk with customer provided keys rather than the default Azure provided encryption keys. Available since v52.0.0. +- **caching** [String, optional]: Type of the disk caching. It can be either `None`, `ReadOnly` or `ReadWrite`. Default is `None`. +- **storage\_account\_type** [String, optional]: Storage account type. Valid only when `use_managed_disks` is `true`. It can be either `Standard_LRS`, `Premium_LRS` or `PremiumV2_LRS`. You can click [**HERE**](http://azure.microsoft.com/en-us/pricing/details/storage/) to learn more about the type of Azure storage account. For `PremiumV2_LRS`, you have to set caching to `None` since `PremiumV2_LRS` does currently not support caching. +- **iops** [Integer, optional]: IOPS of the disk. If you need more IOPS than the baseline offers, you can increase the IOPS of the disks. For more details, see [Premium SSD v2 performance](https://learn.microsoft.com/azure/virtual-machines/disks-types#premium-ssd-v2-performance). Only supported for `PremiumV2_LRS` +- **mbps** [Integer, optional]: Throughput in MB/s of the disk. If you need more throughput than the baseline offers, you can increase the throughput of the disks. For more details, see [Premium SSD v2 performance](https://learn.microsoft.com/azure/virtual-machines/disks-types#premium-ssd-v2-performance). Only supported for `PremiumV2_LRS` +- **disk_encryption_set_name** [String, optional]: The Azure Disk Encryption Set name to use when creating the persistent disk. This is used to encrypt the disk with customer provided keys rather than the default Azure provided encryption keys. Available since v52.0.0. Example of a 10GB Standard LRS disk: @@ -356,43 +380,47 @@ disk_types: To modify the `cloud_properties` such as `storage_account_type`, `iops`, and `mbps` of a disk without the necessity to create a new one and transfer the data, you can utilize the [Native Disk Update Feature](./cpi-api-v2-method/update-disk.md). --- + ## Global Configuration {: #global } Schema: -* **environment** [String, required]: Azure environment name. Possible values are: `AzureCloud`, `AzureChinaCloud`, `AzureUSGovernment` (available in v19+), `AzureGermanCloud` (available in v22+) or `AzureStack`. -* **location** [String, optional]: Azure region name. Only required when `compute_gallery_name` is set, or when [`vm_resources`](https://bosh.io/docs/manifest-v2.html#instance-groups) is specified in the deployment manifest. Available in v33+. -* **subscription_id** [String, required]: Subscription ID. -* **tenant_id** [String, required]: Tenant ID of the service principal. -* **client_id** [String, required]: Client ID of the service principal. -* **client_secret** [String, optional]: Client secret of the service principal. -* **certificate** [String, optional]: The certificate for your service principal. Azure CPI v35.0.0+ supports the [service principal with a certificate](https://github.com/cloudfoundry/bosh-azure-cpi-release/tree/master/docs/advanced/use-service-principal-with-certificate). Only one of `client_secret` and `certificate` can be specified. -* **resource\_group\_name** [String, required]: Resource group name. -* **storage\_account\_name** [String, optional]: Storage account name. It will be used as a default storage account for VM disks and stemcells. If `use_managed_disks` is `false`, `storage_account_name` is required. Otherwise, `storage_account_name` is optional. -* **ssh_user** [String, required]: SSH username. Default: `vcap`. -* **ssh\_public\_key** [String, required]: SSH public key. -* **default\_security\_group** [String, optional]: Name of the default [security group](https://azure.microsoft.com/en-us/documentation/articles/virtual-networks-nsg/) that will be applied to all created VMs. This property is required before v35.0.0, and optional in v35.0.0+. -* **azure_stack** [Hash, optional]: [Configration for AzureStack](https://github.com/cloudfoundry/bosh-azure-cpi-release/tree/master/docs/advanced/azure-stack). Available in v23+. - * **domain** [String, optional]: The domain for your AzureStack deployment. Default is `local.azurestack.external`. You can use the default value for [Azure Stack development kit](https://azure.microsoft.com/en-us/overview/azure-stack/development-kit/). To get this value for Azure Stack integrated systems, contact your service provider. - * **authentication** [String, optional]: The authentication type for your AzureStack deployment. Possible values are: `AzureAD`, `AzureChinaCloudAD` and `ADFS`. You need to specify `certificate` if you select `ADFS`, because Azure Stack with ADFS authentication only supports the service principal with a certificate. - * **resource** [String, optional]: Active Directory Service Endpoint Resource ID, where you can get the token for your AzureStack deployment. - * **endpoint_prefix** [String, optional]: The endpoint prefix for your AzureStack deployment. Default is `management`. - * **skip\_ssl\_validation** [Boolean, optional]: Toggles verification of the Azure Resource Manager REST API SSL certificate. Default is `false`. Deprecated in v35.0.0+. - * **use\_http\_to\_access\_storage\_account** [Boolean, optional]: Flag for using HTTP to access storage account rather than the default HTTPS. Default is `false`. Deprecated in v35.0.0+. - * **ca_cert** [String, required]: All required custom CA certificates for AzureStack. You can [export the Azure Stack CA root certificate](https://docs.microsoft.com/en-us/azure/azure-stack/azure-stack-connect-cli#export-the-azure-stack-ca-root-certificate). Available in v27+. - * The property is required for v35.0.0+. - * For the versions from v27 to v34, if `ca_cert` is not provided, the `skip_ssl_validation` and `use_http_to_access_storage_account` must be set to `true`. -* **parallel\_upload\_thread\_num** [Integer, optional]: The number of threads to upload stemcells in parallel. The default value is 16. -* **debug_mode** [Boolean, optional]: Enable debug mode. The default value is `false`. When `debug_mode` is `true`: - * CPI will log all raw HTTP requests/responses. - * For Azure CPI v26~v35.1.0, the new created VMs (only for VMs in same region with `storage_account_name` specified in [Global Configuration](https://bosh.io/docs/azure-cpi.html#global)) will have [boot diagnostics](https://azure.microsoft.com/en-us/blog/boot-diagnostics-for-virtual-machines-v2/) enabled. **Note**: For Azure CPI v35.2.0+, VM boot diagnostics will NOT be configured by `debug_mode` any more, it will be configured by `enable_vm_boot_diagnostics`. -* **use\_managed\_disks** [Boolean, optional]: Enable managed disks. The default value is `false`. For `AzureCloud`, the option is supported in v21+. For `AzureChinaCloud`, `AzureUSGovernment`, and `AzureGermanCloud`, the option is supported in v26+. For `AzureStack`, the option is not yet supported. -* **pip\_idle\_timeout\_in\_minutes** [Integer, optional]: Set idle timeouts in minutes for dynamic public IPs. It must be in the range [4, 30]. The default value is 4. It is only used when **assign\_dynamic\_public\_ip** is set to `true` in **resource_pool**. Available in V24+. -* **keep\_failed\_vms** [Boolean, optional]: A flag to keep the failed VM. If it's set to `true` and CPI fails to **provision** the VM, CPI will keep the VM for troubleshooting. The default value is `false`. Available in v32+. Please note that the option is different from **keep\_unreachable\_vms** of the [director configuration](https://bosh.io/jobs/director?source=github.com/cloudfoundry/bosh). The latter is to keep the VM whose BOSH agent is unresponsive. -* **enable_telemetry** [Boolean, optional]: A flag to enable telemetry on CPI calls on Azure. Available since v35.2.0. The default value is `true` in v35.2.0, and is `false` in v35.3.0+. -* **enable\_vm\_boot\_diagnostics** [Boolean, optional]: A flag to enable VM boot diagnostics on Azure. Available since v35.2.0. The default value is `true` in v35.2.0, and is `false` in v35.3.0+. -* **compute_gallery_name** [String, optional]: The name of the Azure Compute Gallery to use for managing stemcell images. When provided, the [Compute Gallery feature](./azure-compute-gallery.md) is automatically enabled. Available since v52.0.1+. -* **compute_gallery_replicas** [Integer, optional]: The number of replicas to use for the Compute Gallery Images. Defaults to `3`. Available since v52.0.1+. +- **environment** [String, required]: Azure environment name. Possible values are: `AzureCloud`, `AzureChinaCloud`, `AzureUSGovernment` (available in v19+), `AzureGermanCloud` (available in v22+) or `AzureStack`. +- **location** [String, optional]: Azure region name. Only required when `compute_gallery_name` is set, or when [`vm_resources`](https://bosh.io/docs/manifest-v2.html#instance-groups) is specified in the deployment manifest. Available in v33+. +- **subscription_id** [String, required]: Subscription ID. +- **tenant_id** [String, required]: Tenant ID of the service principal. +- **client_id** [String, required]: Client ID of the service principal. +- **client_secret** [String, optional]: Client secret of the service principal. +- **certificate** [String, optional]: The certificate for your service principal. Azure CPI v35.0.0+ supports the [service principal with a certificate](https://github.com/cloudfoundry/bosh-azure-cpi-release/tree/master/docs/advanced/use-service-principal-with-certificate). Only one of `client_secret` and `certificate` can be specified. +- **resource\_group\_name** [String, required]: Resource group name. +- **storage\_account\_name** [String, optional]: Storage account name. It will be used as a default storage account for VM disks and stemcells. If `use_managed_disks` is `false`, `storage_account_name` is required. Otherwise, `storage_account_name` is optional. +- **ssh_user** [String, required]: SSH username. Default: `vcap`. +- **ssh\_public\_key** [String, required]: SSH public key. +- **default\_security\_group** [String, optional]: Name of the default [security group](https://azure.microsoft.com/en-us/documentation/articles/virtual-networks-nsg/) that will be applied to all created VMs. This property is required before v35.0.0, and optional in v35.0.0+. + +- **azure_stack** [Hash, optional]: [Configration for AzureStack](https://github.com/cloudfoundry/bosh-azure-cpi-release/tree/master/docs/advanced/azure-stack). Available in v23+. + - **domain** [String, optional]: The domain for your AzureStack deployment. Default is `local.azurestack.external`. You can use the default value for [Azure Stack development kit](https://azure.microsoft.com/en-us/overview/azure-stack/development-kit/). To get this value for Azure Stack integrated systems, contact your service provider. + - **authentication** [String, optional]: The authentication type for your AzureStack deployment. Possible values are: `AzureAD`, `AzureChinaCloudAD` and `ADFS`. You need to specify `certificate` if you select `ADFS`, because Azure Stack with ADFS authentication only supports the service principal with a certificate. + - **resource** [String, optional]: Active Directory Service Endpoint Resource ID, where you can get the token for your AzureStack deployment. + - **endpoint_prefix** [String, optional]: The endpoint prefix for your AzureStack deployment. Default is `management`. + - **skip\_ssl\_validation** [Boolean, optional]: Toggles verification of the Azure Resource Manager REST API SSL certificate. Default is `false`. Deprecated in v35.0.0+. + - **use\_http\_to\_access\_storage\_account** [Boolean, optional]: Flag for using HTTP to access storage account rather than the default HTTPS. Default is `false`. Deprecated in v35.0.0+. + - **ca_cert** [String, required]: All required custom CA certificates for AzureStack. You can [export the Azure Stack CA root certificate](https://docs.microsoft.com/en-us/azure/azure-stack/azure-stack-connect-cli#export-the-azure-stack-ca-root-certificate). Available in v27+. + - The property is required for v35.0.0+. + - For the versions from v27 to v34, if `ca_cert` is not provided, the `skip_ssl_validation` and `use_http_to_access_storage_account` must be set to `true`. + +- **parallel\_upload\_thread\_num** [Integer, optional]: The number of threads to upload stemcells in parallel. The default value is 16. +- **debug_mode** [Boolean, optional]: Enable debug mode. The default value is `false`. When `debug_mode` is `true`: + - CPI will log all raw HTTP requests/responses. + - For Azure CPI v26~v35.1.0, the new created VMs (only for VMs in same region with `storage_account_name` specified in [Global Configuration](https://bosh.io/docs/azure-cpi.html#global)) will have [boot diagnostics](https://azure.microsoft.com/en-us/blog/boot-diagnostics-for-virtual-machines-v2/) enabled. **Note**: For Azure CPI v35.2.0+, VM boot diagnostics will NOT be configured by `debug_mode` any more, it will be configured by `enable_vm_boot_diagnostics`. + +- **use\_managed\_disks** [Boolean, optional]: Enable managed disks. The default value is `false`. For `AzureCloud`, the option is supported in v21+. For `AzureChinaCloud`, `AzureUSGovernment`, and `AzureGermanCloud`, the option is supported in v26+. For `AzureStack`, the option is not yet supported. +- **pip\_idle\_timeout\_in\_minutes** [Integer, optional]: Set idle timeouts in minutes for dynamic public IPs. It must be in the range [4, 30]. The default value is 4. It is only used when **assign\_dynamic\_public\_ip** is set to `true` in **resource_pool**. Available in V24+. +- **keep\_failed\_vms** [Boolean, optional]: A flag to keep the failed VM. If it's set to `true` and CPI fails to **provision** the VM, CPI will keep the VM for troubleshooting. The default value is `false`. Available in v32+. Please note that the option is different from **keep\_unreachable\_vms** of the [director configuration](https://bosh.io/jobs/director?source=github.com/cloudfoundry/bosh). The latter is to keep the VM whose BOSH agent is unresponsive. +- **enable_telemetry** [Boolean, optional]: A flag to enable telemetry on CPI calls on Azure. Available since v35.2.0. The default value is `true` in v35.2.0, and is `false` in v35.3.0+. +- **enable\_vm\_boot\_diagnostics** [Boolean, optional]: A flag to enable VM boot diagnostics on Azure. Available since v35.2.0. The default value is `true` in v35.2.0, and is `false` in v35.3.0+. +- **compute_gallery_name** [String, optional]: The name of the Azure Compute Gallery to use for managing stemcell images. When provided, the [Compute Gallery feature](./azure-compute-gallery.md) is automatically enabled. Available since v52.0.1+. +- **compute_gallery_replicas** [Integer, optional]: The number of replicas to use for the Compute Gallery Images. Defaults to `3`. Available since v52.0.1+. See [all configuration options](https://bosh.io/jobs/azure_cpi?source=github.com/cloudfoundry/bosh-azure-cpi-release). @@ -414,6 +442,7 @@ default_security_group: nsg-azure ``` --- + ## Example Cloud Config {: #cloud-config } ```yaml diff --git a/content/azure-managed-identity.md b/content/azure-managed-identity.md index 820244158..719fe0a06 100644 --- a/content/azure-managed-identity.md +++ b/content/azure-managed-identity.md @@ -1,3 +1,5 @@ +# Azure Managed Identity + You can configure BOSH to use [Azure Managed Identities](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/qs-configure-cli-windows-vm) to avoid hard coding specific Azure credentials. Azure Managed Identities are similar to [AWS instance profiles](aws-iam-instance-profiles.md). You first have to create an Azure Managed Identity, and give the proper roles/permissions required by BOSH (i.e. create/delete VM, create/delete/attach disks). diff --git a/content/azure-resources.md b/content/azure-resources.md index 388d5bd28..465ce7c56 100644 --- a/content/azure-resources.md +++ b/content/azure-resources.md @@ -1,3 +1,5 @@ +# Azure - Creating Resources + ## Subscription {: #subscription } To find out subscription and tenant ID use following commands: @@ -33,9 +35,9 @@ Should result in: !!! note If `tenantId` is not present, you may be using a personal account to log in to your Azure subscription. Switch to using work or school account. - * If you are using Azure cloud in China, you should switch the cloud from `AzureCloud` to `AzureChinaCloud`. - * If you are using Azure cloud in Azure Government, you should switch the cloud from `AzureCloud` to `AzureUSGovernment`. - * If you are using Azure cloud in German Cloud, you should switch the cloud from `AzureCloud` to `AzureGermanCloud`. + - If you are using Azure cloud in China, you should switch the cloud from `AzureCloud` to `AzureChinaCloud`. + - If you are using Azure cloud in Azure Government, you should switch the cloud from `AzureCloud` to `AzureUSGovernment`. + - If you are using Azure cloud in German Cloud, you should switch the cloud from `AzureCloud` to `AzureGermanCloud`. Once you've determined your subscription ID, switch to using that account: @@ -52,6 +54,7 @@ az provider register --namespace Microsoft.Compute ``` --- + ## Client {: #client } Azure CPI needs client ID and secret to make authenticated requests. @@ -88,6 +91,7 @@ az role assignment create --role "Contributor" --assignee "http://mycpi" --scope ``` --- + ## Resource Group {: #res-group } Create a resource group in one of the supported [Azure locations](http://azure.microsoft.com/en-us/regions/): @@ -115,6 +119,7 @@ Should result in: Make sure to wait for 'Provisioning State' to become `Succeeded`. --- + ## Virtual Network & Subnet {: #virtual-network } Create a virtual network: @@ -170,6 +175,7 @@ Should result in: ``` --- + ## Network Security Group {: #network-security-group } Create two network security groups: @@ -188,6 +194,7 @@ az network nsg rule create --resource-group bosh-res-group --nsg-name nsg-cf --a ``` --- + ## Public IPs {: #public-ips } To make certain VMs publicly accessible, you will need to create a Public IP. If Azure Availability Zones is used in [AZs](azure-cpi.md#azs), the Public IP should be created with type `Standard SKU`; otherwise, you can use the default `Basic SKU`. @@ -228,6 +235,7 @@ Should result in: You can skip below section if you are using managed disks with Azure CPI v21+ --- + ## Storage Account {: #storage-account } Create a default storage account to hold root disks, persistent disks, stemcells, etc. @@ -333,6 +341,7 @@ Should result in: ``` --- + ### Storage Account Containers {: #storage-account-container } CPI expects to find `bosh` and `stemcell` containers within a default storage account: @@ -391,6 +400,7 @@ Should result in: ``` --- + ### Storage Account Tables {: #storage-account-tables } To support multiple storage accounts, you need to create the following tables in the default storage account: @@ -413,6 +423,7 @@ Should result in: ``` --- + ### Compute Gallery {: #compute-gallery } !!! note diff --git a/content/azure.md b/content/azure.md index 2a3afa8e2..fa10c73cf 100644 --- a/content/azure.md +++ b/content/azure.md @@ -1,30 +1,25 @@ ---- -title: Microsoft Azure ---- - # Microsoft Azure The `azure` CPI can be used with [Microsoft Azure](https://azure.microsoft.com/). - * Release: [cloudfoundry/bosh-azure-cpi-release](https://github.com/cloudfoundry/bosh-azure-cpi-release) - * Issues: [GitHub Issues](https://github.com/cloudfoundry/bosh-azure-cpi-release/issues) - * Slack: [cloudfoundry#bosh-azure-cpi](https://cloudfoundry.slack.com/messages/bosh-azure-cpi) - +- Release: [cloudfoundry/bosh-azure-cpi-release](https://github.com/cloudfoundry/bosh-azure-cpi-release) +- Issues: [GitHub Issues](https://github.com/cloudfoundry/bosh-azure-cpi-release/issues) +- Slack: [cloudfoundry#bosh-azure-cpi](https://cloudfoundry.slack.com/messages/bosh-azure-cpi) ## Concepts The following table maps BOSH concepts to their Azure-native equivalents. -| BOSH | Microsoft Azure | -| ----------------- | -------------------------------------------------- | -| Availability Zone | [Availability Zone][azure_docs_azs] | -| Virtual Machine | [Virtual Machine][azure_docs_vm_sizes] | -| Network Subnet | [Virtual Network Subnet][azure_docs_vnets] | -| Virtual IP | [Public IP][azure_docs_pub_ips] | +| BOSH | Microsoft Azure | +| ----------------- | ------------------------------------------------------------------------------ | +| Availability Zone | [Availability Zone][azure_docs_azs] | +| Virtual Machine | [Virtual Machine][azure_docs_vm_sizes] | +| Network Subnet | [Virtual Network Subnet][azure_docs_vnets] | +| Virtual IP | [Public IP][azure_docs_pub_ips] | | Persistent Disk | [Disk Storage][azure_docs_disks] and [Managed Disks][azure_docs_managed_disks] | -| Disk Snapshot | [Managed Disk Snapshot][azure_docs_disk_snapshots] | -| Stemcell | Disk Storage Blobs and Managed Disk Blobs | -| Agent Settings | Config Drive; BOSH Registry | +| Disk Snapshot | [Managed Disk Snapshot][azure_docs_disk_snapshots] | +| Stemcell | Disk Storage Blobs and Managed Disk Blobs | +| Agent Settings | Config Drive; BOSH Registry | [azure_docs_azs]: https://docs.microsoft.com/en-us/azure/availability-zones/az-overview [azure_docs_vm_sizes]: https://docs.microsoft.com/en-us/azure/virtual-machines/linux/sizes diff --git a/content/basic-workflow.md b/content/basic-workflow.md index 330a57add..fa8c0bd0d 100644 --- a/content/basic-workflow.md +++ b/content/basic-workflow.md @@ -1,3 +1,5 @@ +# Basic Workflow + (Follow [Create an environment?](init.md) to create the Director.) The Director can manage multiple [deployments](terminology.md#deployment). diff --git a/content/blobstore-agent-password-rotation.md b/content/blobstore-agent-password-rotation.md index ca0ff0ada..e59ad1d8a 100644 --- a/content/blobstore-agent-password-rotation.md +++ b/content/blobstore-agent-password-rotation.md @@ -1,18 +1,18 @@ +# Agent Password Rotation + !!! note Applicable for director version 268.5.0+ -# Rotating Blobstore Agent Password - As of director version 268.5.0+, blobstore agent password can be rotated. -### Preconditions +## Preconditions -* The Director is in a healthy state. -* Take note of any ignored VMs. They will be omitted from the VM recreation steps. -* All the VMs are in the `running` state in all deployments. -* These instructions must be adapted if used with ops files (i.e. bosh-lite), as they overwrite the variables used in this procedure. +- The Director is in a healthy state. +- Take note of any ignored VMs. They will be omitted from the VM recreation steps. +- All the VMs are in the `running` state in all deployments. +- These instructions must be adapted if used with ops files (i.e. bosh-lite), as they overwrite the variables used in this procedure. -### Step 1: Update the director to add a new user {: #step-1} +## Step 1: Update the director to add a new user {: #step-1} ```shell OLD_PWD=$(bosh interpolate --path=/blobstore_agent_password creds.yml) @@ -64,10 +64,10 @@ Ops file `rotate-blobstore-agent-password.yml`: password: ((blobstore_agent_old_password)) ``` -* move the old user and password to `additional_users` section of blobstore properties -* create new user and password +- move the old user and password to `additional_users` section of blobstore properties +- create new user and password -### Step 2: Recreate all VMs {: #step-2} +## Step 2: Recreate all VMs {: #step-2} The recreation of all VMs will add the new credentials and remove the old credentials from their agent settings. @@ -76,7 +76,7 @@ settings. bosh -d deployment-name recreate ``` -### Step 3: Update director to remove old user {: #step-3} +## Step 3: Update director to remove old user {: #step-3} ```shell bosh interpolate ./creds.yml \ @@ -115,9 +115,9 @@ Ops file `rename-default-agent-user.yml`: value: agent-new ``` -* Deploy director without the old user in the `additional_users` section of blobstore properties. -* Since the new user name is `agent-new`, from now on you have to deploy the Director with the ops file `rename-default-agent-user.yml`. +- Deploy director without the old user in the `additional_users` section of blobstore properties. +- Since the new user name is `agent-new`, from now on you have to deploy the Director with the ops file `rename-default-agent-user.yml`. -### Optional steps: +## Optional steps Rotate a second time to get rid of the additional `rename-default-agent-user.yml` ops file by naming the new user back to `agent`. diff --git a/content/blobstore-ca-rotation.md b/content/blobstore-ca-rotation.md index 194c5d320..43939e702 100644 --- a/content/blobstore-ca-rotation.md +++ b/content/blobstore-ca-rotation.md @@ -10,9 +10,9 @@ This procedure works by deploying both the old and the new CA on all the VMs in ### Preconditions -* The Director is in a healthy state. -* All the VMs are in the `running` state in all deployments. -* Director versions prior to 271.12 and stemcells prior to Bionic 1.36 and Windows 2019.41 need to recreate VMs as part of the redeploy steps. +- The Director is in a healthy state. +- All the VMs are in the `running` state in all deployments. +- Director versions prior to 271.12 and stemcells prior to Bionic 1.36 and Windows 2019.41 need to recreate VMs as part of the redeploy steps. ### Step 1: Redeploy the director with the new blobstore CA. {: #step-1} @@ -26,10 +26,10 @@ bosh create-env ~/workspace/bosh-deployment/bosh.yml \ -v ... additional vars ``` -* This adds new variables for the new CA/certificates/private_key. -* The director is given a modified CA with the original CA and the new CA concatenated as `((blobstore_server_tls.ca))((blobstore_server_tls_2.ca))`. -* The blobstore continues to use the old certificates and private key. -* Each VM/agent continues to use the old certificates to communicate with the blobstore. +- This adds new variables for the new CA/certificates/private_key. +- The director is given a modified CA with the original CA and the new CA concatenated as `((blobstore_server_tls.ca))((blobstore_server_tls_2.ca))`. +- The blobstore continues to use the old certificates and private key. +- Each VM/agent continues to use the old certificates to communicate with the blobstore. `add-new-blobstore-ca.yml` @@ -81,9 +81,9 @@ bosh create-env ~/workspace/bosh-deployment/bosh.yml \ `remove-old-blobstore-ca.yml` -* `remove-old-blobstore-ca` below is used to remove the old CA from the concatenated CAs. -* The blobstore server is updated to use a new certificate and private key, generated by the new CA. -* All the components can now communicate using the new CA. +- `remove-old-blobstore-ca` below is used to remove the old CA from the concatenated CAs. +- The blobstore server is updated to use a new certificate and private key, generated by the new CA. +- All the components can now communicate using the new CA. ```yaml --- @@ -130,8 +130,7 @@ the above opsfiles for subsequent `bosh create-env` commands. This is not ideal The following opsfile in conjunction with the `bosh interpolate` command will replace the old certificate values with the new, and remove the second variable created for the above process. - -``` +```yaml # update_blobstore_var_values.yml --- @@ -153,7 +152,7 @@ certificate values with the new, and remove the second variable created for the Create a backup of the credentials and apply the opsfile: -``` +```bash cp creds.yml creds.yml.backup bosh interpolate creds.yml \ diff --git a/content/bosh-cli.md b/content/bosh-cli.md index a20d7472b..f151b3a34 100644 --- a/content/bosh-cli.md +++ b/content/bosh-cli.md @@ -1,3 +1,5 @@ +# CLI v1 Installation + BOSH Command Line Interface (CLI) is used to interact with the Director. The CLI is written in Ruby and is distributed via `bosh_cli` gem. ```shell @@ -9,7 +11,7 @@ gem install bosh_cli --no-ri --no-rdoc If gem installation does not succeed, make sure pre-requisites for your OS are met. -### Prerequisites on Ubuntu Trusty +## Prerequisites on Ubuntu Trusty Make sure following packages are installed: @@ -19,7 +21,7 @@ sudo apt-get install build-essential ruby ruby-dev libxml2-dev libsqlite3-dev li Make sure `ruby` and `gem` binaries are on your path before continuing. -### Prerequisites on CentOS +## Prerequisites on CentOS Make sure following packages are installed: @@ -27,18 +29,22 @@ Make sure following packages are installed: sudo yum install gcc ruby ruby-devel mysql-devel postgresql-devel postgresql-libs sqlite-devel libxslt-devel libxml2-devel yajl-ruby ``` -### Prerequisites on Mac OS X +## Prerequisites on Mac OS X You may see an error like this: +```text ERROR: While executing gem ... (Gem::FilePermissionError) You don't have write permissions for the /Library/Ruby/Gems/2.0.0 directory. +``` Instead of using the system Ruby, install a separate Ruby for your own use, and switch to that one using a package like RVM, rbenv, or chruby. You may see an error like this: +```text The compiler failed to generate an executable file. (RuntimeError). You have to install development tools first. +``` Make sure you have installed Xcode and the command-line developer tools, and agreed to the license. @@ -49,7 +55,9 @@ xcode-select --install A window will pop up saying: +```text The "xcode-select" command requires the command line developer tools. Would you like to install the tools now? +``` Choose Install to continue. Choose Get Xcode to install Xcode and the command line developer tools from the App Store. If you have already installed Xcode from the App Store, you can choose Install and it will install the cli tools. diff --git a/content/bosh-components.md b/content/bosh-components.md index 9ad643e12..624c4cbc4 100644 --- a/content/bosh-components.md +++ b/content/bosh-components.md @@ -1,8 +1,11 @@ +# Components of BOSH + Before creating a new [environment](terminology.md#environment) we recommend to learn the names of major components that will be installed, used and configured: ![image](images/bosh-architecture.png) --- + ## Command Line Interface (CLI) {: #cli } The Command Line Interface (CLI) is the primary operator interface to BOSH. An operator uses the CLI to interact with the Director and perform actions on the cloud. @@ -10,15 +13,16 @@ The Command Line Interface (CLI) is the primary operator interface to BOSH. An o CLI is typically installed on a machine that can directly communicate with the Director's API, e.g. an operator's laptop, or a jumpbox in the datacenter. --- + ## Director {: #director } The Director is the core orchestrating component in BOSH. The Director controls VM creation and deployment, as well as other software and service lifecycle events. The Director creates actionable tasks: -* By translating commands sent by an operator through the CLI -* From scheduled processes like backups or snapshots -* If needed to reconcile the expected state with the actual state of a VM +- By translating commands sent by an operator through the CLI +- From scheduled processes like backups or snapshots +- If needed to reconcile the expected state with the actual state of a VM Once created, the Director adds these tasks to the Task Queue. Worker processes take tasks from the Task Queue and act on them. @@ -35,6 +39,7 @@ Director workers take tasks from the Task Queue and act on them. A Cloud Provider Interface (CPI) is an API that the Director uses to interact with an IaaS to create and manage stemcells, VMs, and disks. A CPI abstracts infrastructure differences from the rest of BOSH. --- + ## Health Monitor {: #health-monitor } The Health Monitor uses status and lifecycle events received from Agents to monitor the health of VMs. If the Health Monitor detects a problem with a VM, it can send an alert through notification plugins, or trigger the Resurrector. @@ -44,6 +49,7 @@ The Health Monitor uses status and lifecycle events received from Agents to moni If enabled, the Resurrector plugin automatically recreates VMs identified by the Health Monitor as missing or unresponsive. It uses same Director API that CLI uses. --- + ## Components used to store Director's persistent data {: #persistent } ### Database {: #database } @@ -55,6 +61,7 @@ The Director uses a Postgres database to store information about the desired sta The Blobstore stores the source forms of releases and the compiled images of releases. An operator uploads a release using the CLI, and the Director inserts the release into the Blobstore. When you deploy a release, BOSH orchestrates the compilation of packages and stores the result in the Blobstore. --- + ## Agent {: #agent } BOSH includes an Agent on every VM that it deploys. The Agent listens for instructions from the Director and carries out those instructions. The Agent receives job specifications from the Director and uses them to assign a role, or Job, to the VM. @@ -62,6 +69,7 @@ BOSH includes an Agent on every VM that it deploys. The Agent listens for instru For example, to assign the job of running MySQL to a VM, the Director sends instructions to the Agent on the VM. These instructions include which packages to install and how to configure those packages. The Agent uses these instructions to install and configure MySQL on the VM. --- + ## Components used for cross-component communication {: #comm } ### Message Bus (NATS) {: #nats } @@ -69,11 +77,13 @@ For example, to assign the job of running MySQL to a VM, the Director sends inst The Director and the Agents communicate through a lightweight publish-subscribe messaging system called NATS. These messages have two purposes: to perform provisioning instructions on the VMs, and to inform the Health Monitor about changes in the health of monitored processes. --- + ## UAA {: #dns } The User Account and Authentication Server is primarily used as OAuth2 provider used to issue OAuth tokens. [See more details.](https://docs.cloudfoundry.org/concepts/architecture/uaa.html) --- + ## Example component interaction {: #example } This example shows how components interact when creating a new VM. diff --git a/content/bosh-lite.md b/content/bosh-lite.md index 0aa53dc72..6a3997f07 100644 --- a/content/bosh-lite.md +++ b/content/bosh-lite.md @@ -1,3 +1,5 @@ +# VirtualBox + BOSH Lite v2 is a Director VM running in VirtualBox (typically locally). It is managed via [CLI v2](cli-v2.md). Internally CPI uses containers to emulate VMs which makes it an excellent choice for: - General BOSH exploration without investing time and resources to configure an IaaS @@ -5,6 +7,7 @@ BOSH Lite v2 is a Director VM running in VirtualBox (typically locally). It is m - Testing releases locally or in CI --- + ## Install {: #install } Follow below steps to get it running on locally on VirtualBox: @@ -85,8 +88,8 @@ Follow below steps to get it running on locally on VirtualBox: route add 10.244.0.0/16 192.168.56.6 # Windows ``` - --- + ## Deploy example Zookeeper deployment {: #deploy } Run through quick steps below or follow [deploy workflow](basic-workflow.md) that goes through the same steps but with more explanation. @@ -117,8 +120,7 @@ Run through quick steps below or follow [deploy workflow](basic-workflow.md) tha bosh -e vbox -d zookeeper run-errand smoke-tests ``` - ## Tips {: #tips } -* In case you need to SSH into the Director VM, see [Jumpbox](jumpbox.md). -* In case VirtualBox VM shuts down or reboots, you will have to re-run `create-env` command from above with `--recreate` flag. The containers will be lost after a VM restart, but you can restore your deployment with `bosh cck` command. Alternatively *Pause* the VM from the VirtualBox UI before shutting down VirtualBox host, or making your computer sleep. +- In case you need to SSH into the Director VM, see [Jumpbox](jumpbox.md). +- In case VirtualBox VM shuts down or reboots, you will have to re-run `create-env` command from above with `--recreate` flag. The containers will be lost after a VM restart, but you can restore your deployment with `bosh cck` command. Alternatively *Pause* the VM from the VirtualBox UI before shutting down VirtualBox host, or making your computer sleep. diff --git a/content/build-cpi.md b/content/build-cpi.md index 82b8b3c36..44a700567 100644 --- a/content/build-cpi.md +++ b/content/build-cpi.md @@ -1,3 +1,5 @@ +# Building a CPI + This topic describes how to build a CPI. ## Distribution {: #distribution } @@ -13,33 +15,29 @@ release, it must include a release job that has `bin/cpi` executable. Both `bosh create-env` command and the Director expect to be configured with a CPI release to function properly. In the case of `bosh create-env` command, specified CPI release is unpacked and installed on the machine running the command. For the Director, CPI release job is colocated on the same VM, so that the director release job can access it. - ## Implementation When building a CPI release, the primary requirement is that it provides a `bin/cpi` executable which implements a simple RPC API through `STDIN`/`STDOUT`. The [RPC API](cpi-api-rpc.md) page provides an in-depth look at the protocol and required methods. If you are getting started with a new CPI, you may be interested in using one of the following languages. These releases take advantage of some existing libraries that you may find useful in your own implementation. - ### Ruby The [`bosh_cpi`](https://rubygems.org/gems/bosh_cpi) gem provides a `Bosh::Cpi::Cli` class which handles the deserialization and serialization of the RPC calls. You can see examples of this in the following CPIs: - * [Amazon Web Services CPI Release](https://github.com/cloudfoundry/bosh-aws-cpi-release) - * [Microsoft Azure CPI Release](https://github.com/cloudfoundry/bosh-azure-cpi-release) - * [OpenStack CPI Release](https://github.com/cloudfoundry/bosh-openstack-cpi-release) - * [VMware vSphere CPI Release](https://github.com/cloudfoundry/bosh-vsphere-cpi-release) - +- [Amazon Web Services CPI Release](https://github.com/cloudfoundry/bosh-aws-cpi-release) +- [Microsoft Azure CPI Release](https://github.com/cloudfoundry/bosh-azure-cpi-release) +- [OpenStack CPI Release](https://github.com/cloudfoundry/bosh-openstack-cpi-release) +- [VMware vSphere CPI Release](https://github.com/cloudfoundry/bosh-vsphere-cpi-release) ### Go There are a few CPI releases written in Go, as well: - * [Alibaba Cloud CPI Release](https://github.com/cloudfoundry-incubator/bosh-alicloud-cpi-release) - * [Google CPI Release](https://github.com/cloudfoundry/bosh-google-cpi-release) - * [VirtualBox CPI Release](https://github.com/cloudfoundry/bosh-virtualbox-cpi-release) - * [Warden CPI Release](https://github.com/cloudfoundry/bosh-warden-cpi-release) - +- [Alibaba Cloud CPI Release](https://github.com/cloudfoundry-incubator/bosh-alicloud-cpi-release) +- [Google CPI Release](https://github.com/cloudfoundry/bosh-google-cpi-release) +- [VirtualBox CPI Release](https://github.com/cloudfoundry/bosh-virtualbox-cpi-release) +- [Warden CPI Release](https://github.com/cloudfoundry/bosh-warden-cpi-release) ## Testing {: #testing } @@ -48,7 +46,6 @@ There are two test suites each CPI is expected to pass before it's considered to - its own [CPI Lifecycle Tests](https://github.com/cloudfoundry/bosh/blob/master/docs/running_tests.md#cpi-lifecycle-tests) which should provide integration level coverage for each CPI method - shared [BOSH Acceptance Tests (BATS)](https://github.com/cloudfoundry/bosh/blob/master/docs/running_tests.md#bosh-acceptance-tests-bats) (provided by the BOSH team) which verify high level Director behavior with the CPI activated - ## Concurrency {: #concurrency } The CPI is expected to handle multiple method calls concurrently (and in parallel) with a promise that arguments represent different IaaS resources. For example, multiple `create_vm` CPI method calls may be issued that all use the same stemcell cloud ID; however, `attach_disk` CPI method will never be called with the same VM cloud ID concurrently. @@ -57,12 +54,10 @@ The CPI is expected to handle multiple method calls concurrently (and in paralle Since each CPI method call is a separate OS process, simple locking techniques (Ruby's `Mutex.new` for example) will not work. - ## Rate Limiting {: #rate-limiting } Most CPIs have to deal with IaaS APIs that rate limit (e.g. OpenStack, AWS). Currently it is the responsibility of the CPI to handle rate-limiting errors, properly catch them, wait and retry actions that were interrupted. Given that there is no timeout around how long a CPI method can run, it's all right to wait as long as necessary to resume making API calls. Though it's suggested to log such information. - ## Debugging {: #debugging } It is usually useful to get a detailed log of CPI requests and responses from the callee. To get a full debug log from `bosh create-env` command set `BOSH_LOG_LEVEL=debug` environment variable. diff --git a/content/build-stemcell.md b/content/build-stemcell.md index 3774d0f48..558df0376 100644 --- a/content/build-stemcell.md +++ b/content/build-stemcell.md @@ -1,3 +1,5 @@ +# Building a Stemcell + (See [What is a Stemcell?](stemcell.md) for an introduction to stemcells.) To build a stemcell tarball for a supported IaaS-OS combination follow instructions in the [bosh-linux-stemcell-builder's README](https://github.com/cloudfoundry/bosh-linux-stemcell-builder/blob/master/README.md). @@ -14,6 +16,7 @@ In the future, BOSH team will investigate how to best consolidate stemcells into - [bosh-agent](https://github.com/cloudfoundry/bosh-agent) --- + ## Tarball Structure {: #tarball-structure } !!! info @@ -32,24 +35,24 @@ Should result in: -rw-r--r-- 0 ubuntu ubuntu 12543 Aug 4 09:22 dev_tools_file_list.txt ``` -* `image`: OS image in a format (raw, qcow, ova, etc.) understood by the CPI/IaaS. -* `stemcell.MF`: YAML file with stemcell metadata. -* `packages.txt`: Text file that includes list of packages installed. (Used to be included as `stemcell_dpkg_l.txt`) -* `dev_tools_file_list.txt`: Text file that includes list of files removed by the agent if Agent's `remove_dev_tools` feature is enabled. +- `image`: OS image in a format (raw, qcow, ova, etc.) understood by the CPI/IaaS. +- `stemcell.MF`: YAML file with stemcell metadata. +- `packages.txt`: Text file that includes list of packages installed. (Used to be included as `stemcell_dpkg_l.txt`) +- `dev_tools_file_list.txt`: Text file that includes list of files removed by the agent if Agent's `remove_dev_tools` feature is enabled. ### Metadata {: #metadata } !!! info This is an implementation detail. The content of `stemcell.MF` is subject to change without notice. -* **name** [String, required]: A unique name used to identify stemcell series. -* **operating_system** [String, required]: Operating system in the stemcell. Example: `ubuntu-xenial`. -* **version** [String, required]: Version of the stemcell. Example: `621.74`. -* **sha1** [String, required]: The SHA1 of the image file included in the stemcell tarball. -* **bosh_protocol** [String, optional]: Deprecated. -* **cloud_properties** [Hash, required]: Describes any IaaS-specific properties needed to import OS image. These properties will be passed in to the [`create_stemcell` CPI call](cpi-api-v2.md#create-stemcell). -* **api_version** [Integer, optional]: Highest supported API version of the Agent in the stemcell. Defaults to `1`. -* **stemcell_formats** [Array of Strings, optional]: The list of stemcell formats that a [CPI must support](cpi-api-v2-method/info.md#result). The director will attempt to upload the stemcell to all CPIs that support any specified formats. +- **name** [String, required]: A unique name used to identify stemcell series. +- **operating_system** [String, required]: Operating system in the stemcell. Example: `ubuntu-xenial`. +- **version** [String, required]: Version of the stemcell. Example: `621.74`. +- **sha1** [String, required]: The SHA1 of the image file included in the stemcell tarball. +- **bosh_protocol** [String, optional]: Deprecated. +- **cloud_properties** [Hash, required]: Describes any IaaS-specific properties needed to import OS image. These properties will be passed in to the [`create_stemcell` CPI call](cpi-api-v2.md#create-stemcell). +- **api_version** [Integer, optional]: Highest supported API version of the Agent in the stemcell. Defaults to `1`. +- **stemcell_formats** [Array of Strings, optional]: The list of stemcell formats that a [CPI must support](cpi-api-v2-method/info.md#result). The director will attempt to upload the stemcell to all CPIs that support any specified formats. Name, operating system and version values will be visible via `bosh stemcells` command once a stemcell is imported into the Director. @@ -84,6 +87,7 @@ cloud_properties: ``` --- + ## Light Stemcells {: #light-stemcells } A "light" stemcell represents a reference to an IaaS resource where the stemcell has already been imported. This helps solve IaaS limitations which restrict how base VM images can be imported, such as: @@ -132,7 +136,6 @@ cloud_properties: cn-north-1: ami-01db1b9ef2de116fb ``` - ### Publishing {: #light-stemcell-publishing } The process of building light stemcells will depend on your IaaS. Typically, you will have an automation pipeline which does the following: @@ -144,16 +147,17 @@ The process of building light stemcells will depend on your IaaS. Typically, you If you're getting started in this process, you may want to refer to the following examples: - * Amazon Web Services ([cloudfoundry/bosh-aws-light-stemcell-builder](https://github.com/cloudfoundry/bosh-aws-light-stemcell-builder)) - * Google Cloud Platform ([cloudfoundry/bosh-google-light-stemcell-builder](https://github.com/cloudfoundry/bosh-google-light-stemcell-builder)) - * OpenStack ([docs](openstack-light-stemcells.md)) - this uses the `repack-stemcell` command of the CLI +- Amazon Web Services ([cloudfoundry/bosh-aws-light-stemcell-builder](https://github.com/cloudfoundry/bosh-aws-light-stemcell-builder)) +- Google Cloud Platform ([cloudfoundry/bosh-google-light-stemcell-builder](https://github.com/cloudfoundry/bosh-google-light-stemcell-builder)) +- OpenStack ([docs](openstack-light-stemcells.md)) - this uses the `repack-stemcell` command of the CLI While the import step is highly IaaS-specific, there are a couple general recommendations: - * You may want to reuse the process that the CPI uses internally with `create_stemcell`. If not reusing the same code, you should follow the exact same steps. There should be no noticeable difference between an IaaS base image created from a "light stemcell builder" vs an operator importing the stemcell on a director themselves. - * Your IaaS may support alternative methods for transferring images once they're imported. As long as the process does not change the underlying stemcell image, you may feel free to use it. For example, in AWS we import the image into a single region, then use the `CopyImage` AWS API call to copy the image to all other regions. +- You may want to reuse the process that the CPI uses internally with `create_stemcell`. If not reusing the same code, you should follow the exact same steps. There should be no noticeable difference between an IaaS base image created from a "light stemcell builder" vs an operator importing the stemcell on a director themselves. +- Your IaaS may support alternative methods for transferring images once they're imported. As long as the process does not change the underlying stemcell image, you may feel free to use it. For example, in AWS we import the image into a single region, then use the `CopyImage` AWS API call to copy the image to all other regions. --- + ## Testing {: #testing } There are two test suites each stemcell is expected to pass before it's considered to be production-ready: diff --git a/content/cck.md b/content/cck.md index 4f30bc2c5..c4598fada 100644 --- a/content/cck.md +++ b/content/cck.md @@ -1,3 +1,5 @@ +# IaaS Reconciliation + !!! Note Updated for bosh-release v183 (1.3010.0). @@ -52,6 +54,7 @@ No problems found ``` --- + ## Problems {: #problems } ### VM is missing {: #missing-vm } @@ -134,6 +137,7 @@ Cloudcheck is finished In the above example options 3 was picked and VM reference was deleted. --- + ### VM is not responsive (unresponsive agent) {: #not-responsive-vm } Assuming there was a deployment with a VM, somehow Agent is no longer @@ -355,6 +359,7 @@ cck determined that `vol-549f071f` persistent disk is not attached to `i-4fcd99b all release job processes. --- + ### Inactive Disk {: #inactive-disk } Assuming there are are disks that are marked as inactive, @@ -423,6 +428,7 @@ cck determined that `disk-eaca0b50-daf5-4fba-6dbf-06354e11e0af` persistent disk active VM already has a disk before marking the disk as active --- + ### Persistent Disk is missing {: #missing-persistent-disk } Assuming there was a deployment with a VM, somehow persistent disk got deleted. @@ -436,26 +442,18 @@ Note: Not all CPIs implement needed functionality to determine if disk is missin Automate recovery using the `--resolution=RESOLUTION-VALUE` cloud-check (cck) option, where `RESOLUTION-VALUE` represents one of the following: - * `ignore` - skip resolution - * `recreate_vm` - recreate VM and wait for processes to start - * `recreate_vm_without_wait` - recreate VM without waiting for processes to - start (and thus [`post-start` scripts](job-lifecycle.md#start) won't run - after processes have restarted, which could have consequences depending on - how critical `post-start` scripts are) - * `reboot_vm` - reboot the VM - * `delete_vm` - delete the VM - * `delete_vm_reference` - remove the VM reference that Director has (this - could cause IaaS resources to be abandoned) - * `delete_disk` - delete the disk - * `delete_disk_reference` - remove the disk reference that Director has (this +- `ignore` - skip resolution +- `recreate_vm` - recreate VM and wait for processes to start +- `recreate_vm_without_wait` - recreate VM without waiting for processes to start (and thus [`post-start` scripts](job-lifecycle.md#start) won't run after processes have restarted, which could have consequences depending on how critical `post-start` scripts are) +- `reboot_vm` - reboot the VM +- `delete_vm` - delete the VM +- `delete_vm_reference` - remove the VM reference that Director has (this could cause IaaS resources to be abandoned) - * `activate_disk` - mark the disk as active - * `reattach_disk` - reattach persistent disk to the VM and mount it at its - usual location `/var/vcap/store` - * `reattach_disk_and_reboot`- reattach the persistent disk to the VM and - reboot it so that Agent can safely mount persistent disk before starting - any release job processes. cck will not wait until VM reboots and restarts - all release job processes +- `delete_disk` - delete the disk +- `delete_disk_reference` - remove the disk reference that Director has (this could cause IaaS resources to be abandoned) +- `activate_disk` - mark the disk as active +- `reattach_disk` - reattach persistent disk to the VM and mount it at its usual location `/var/vcap/store` +- `reattach_disk_and_reboot`- reattach the persistent disk to the VM and reboot it so that Agent can safely mount persistent disk before starting any release job processes. cck will not wait until VM reboots and restarts all release job processes !!! Warning Consider using `cck` interactively because the selected resolution will be diff --git a/content/changing-deployment-vm-strategy.md b/content/changing-deployment-vm-strategy.md index 021b0e6e4..d097bbbbc 100644 --- a/content/changing-deployment-vm-strategy.md +++ b/content/changing-deployment-vm-strategy.md @@ -15,7 +15,6 @@ The new strategy is called `create-swap-delete`. This strategy shifts IaaS-relat You can see that `create-swap-delete` introduces new functionality for deferring VM deletions as well. Old VMs will be "orphaned" (similar to disks) and scheduled for clean up. Typically cleanup starts within 5 minutes. - ## Usage To change this behavior in your deployment, add `vm_strategy` to your deployment's [`update` section](manifest-v2.md#update). For example... @@ -28,14 +27,13 @@ To change this behavior in your deployment, add `vm_strategy` to your deployment !!! tip The `update` section can be overridden at the instance group level. This allows you to opt-in or opt-out specific instance groups which need different strategies. - ## Use Cases - * Reduce Deployment Time - when you use the `create-swap-delete` strategy BOSH creates VMs in parallel at the start of the deploy, which will reduce the time taken for VMs to be created or recreated. - * H/A Replacement - you may want to use `create-swap-delete` with a non-H/A deployment due to the reduced update time and downtime (~20s), instead of running a full H/A deployment with either strategy. - * Persistent Disks - if you are using a persistent disk and considering using `create-swap-delete` with a non-H/A deployment, the downtime will also depend on the time taken for the IaaS to detach and attach the persistent disks. IaaSes take different periods of time for this operation, and duration can vary even within the same IaaS. +* Reduce Deployment Time - when you use the `create-swap-delete` strategy BOSH creates VMs in parallel at the start of the deploy, which will reduce the time taken for VMs to be created or recreated. +* H/A Replacement - you may want to use `create-swap-delete` with a non-H/A deployment due to the reduced update time and downtime (~20s), instead of running a full H/A deployment with either strategy. + * Persistent Disks - if you are using a persistent disk and considering using `create-swap-delete` with a non-H/A deployment, the downtime will also depend on the time taken for the IaaS to detach and attach the persistent disks. IaaSes take different periods of time for this operation, and duration can vary even within the same IaaS. ## Caveats - * The `create-swap-delete` strategy will not be used for instances in instance groups that are using static IPs due to the exclusivity of the IPs in IaaSes. - * When using `create-swap-delete`, your IaaS resource usage will inherently surge during the deploy while BOSH creates additional VMs in preparation for update. You may need to review any resource limits which are in effect for your IaaS and account. +* The `create-swap-delete` strategy will not be used for instances in instance groups that are using static IPs due to the exclusivity of the IPs in IaaSes. +* When using `create-swap-delete`, your IaaS resource usage will inherently surge during the deploy while BOSH creates additional VMs in preparation for update. You may need to review any resource limits which are in effect for your IaaS and account. diff --git a/content/cli-env-deps.md b/content/cli-env-deps.md index d88ff4f20..2f8afe886 100644 --- a/content/cli-env-deps.md +++ b/content/cli-env-deps.md @@ -1,3 +1,5 @@ +# CLI env Dependencies + !!! note Applies to CLI v2. @@ -22,11 +24,13 @@ The `bosh create-env` (and `bosh delete-env`) command has dependencies. **Mac OS X** Install Apple Command Line Tools: + ```shell xcode-select --install ``` Use [Homebrew](http://brew.sh) to install OpenSSL: + ```shell brew install openssl ``` diff --git a/content/cli-envs.md b/content/cli-envs.md index 8aab33e56..a7a6531ff 100644 --- a/content/cli-envs.md +++ b/content/cli-envs.md @@ -1,3 +1,5 @@ +# Environments + !!! note Applies to CLI v2. @@ -100,6 +102,7 @@ bosh -e aws env Alternatively you can set `export BOSH_ENVIRONMENT=aws` once instead of using `--environment` flag for each command. --- + ## Deployment State {: #deployment-state } `bosh create-env` command needs to remember resources it creates in the IaaS so that it can re-use or delete them at a later time. The deploy command stores current state of your deployment in a given state file (via `--state` flag) or implicitly in `-state.json` file in the same directory as your deployment manifest. @@ -131,6 +134,7 @@ If for some reason you've lost your deployment state file, or have not saved the 1. Save the deployment state file. --- + ## `delete-env` command {: #delete-env } `bosh delete-env` command idempotently deletes all previously created IaaS resources (VMs, disks, and stemcells). The command will try its best to not return an error, for example it ignores resources that were already deleted and retries on certain operations. @@ -176,6 +180,7 @@ Finished deleting deployment (00:00:04) ``` --- + ## `stop-env` command {: #stop-env } !!! note @@ -209,6 +214,7 @@ Succeeded ``` --- + ## `start-env` command {: #start-env } !!! note diff --git a/content/cli-global-flags.md b/content/cli-global-flags.md index d5f5a7a28..544d28482 100644 --- a/content/cli-global-flags.md +++ b/content/cli-global-flags.md @@ -1,3 +1,5 @@ +# Global Flags + !!! note Applies to CLI v2. @@ -7,6 +9,7 @@ - `bosh -h` shows command specific options --- + ## Version flag {: #version } - `-v` flag shows CLI version. @@ -15,6 +18,7 @@ To see Director version use `bosh env` command. --- + ## Environment flags {: #env } - `--environment` (`-e`) flag allows to specify Director VM address or environment alias (`BOSH_ENVIRONMENT` environment variable) @@ -25,6 +29,7 @@ CLI does not provide a way to skip SSL certificate validation to encourage secur See [CLI environments](cli-envs.md) for details. --- + ## Authentication flags {: #auth } - `--client` flag allows to specify basic auth username or UAA client ID (`BOSH_CLIENT` environment variable) @@ -33,6 +38,7 @@ See [CLI environments](cli-envs.md) for details. CLI does not provide a way to specify UAA user login information since all non-interactive use (in scripts) should use UAA clients. `bosh log-in` command allows to log in interactively as a UAA user. --- + ## Output flags {: #output } - `-n` flag affirms any confirmation that typically requires user input (`BOSH_NON_INTERACTIVE=true` environment variable) @@ -43,6 +49,7 @@ CLI does not provide a way to specify UAA user login information since all non-i CLI makes a distinction between decorative text (table headings) and primary content (such as tables). To make it eas easy to parse command output via other tools (such as grep) when decorative text is automatically hidden when command output is redirected. --- + ## Deployment flag {: #deployment } - `--deployment` (`-d`) flag allows to specify deployment for a command (`BOSH_DEPLOYMENT` environment variable) @@ -50,22 +57,25 @@ CLI makes a distinction between decorative text (table headings) and primary con Several commands that can operate in a Director and a deployment context (such as `bosh tasks` command) account for presence of this flag and filter their output based on a deployment. --- + ## SOCKS5 Tunneling {: #tunnel } See [tunneling](cli-tunnel.md) for details. --- + ## HTTP proxy {: #http-proxy } Some commands such as `create-env` may require usage of an http proxy to access internet or the iaas. Define `HTTP_PROXY`, `HTTPS_PROXY` and `NO_PROXY` environment variables according to [go-lang support](https://golang.org/pkg/net/http/#ProxyFromEnvironment) documented in detail in [x/net/http/httpproxy#Config](https://pkg.go.dev/golang.org/x/net/http/httpproxy#Config) -``` +```shell export HTTP_PROXY=http://myproxy.com:3128 export HTTPS_PROXY=http://myproxy.com:3128 export NO_PROXY="asinglehost.mydomain.com,mywholedomain.org,.onlymysubdomains.org,192.168.10.11,172.17.11.0/24" ``` --- + ## Logging {: #logging } Along with the UI output (stdout) and UI errors (stderr), CLI can output more verbose logs. diff --git a/content/cli-help.md b/content/cli-help.md deleted file mode 100644 index 581576b71..000000000 --- a/content/cli-help.md +++ /dev/null @@ -1,4 +0,0 @@ -!!! note - Examples require CLI v2. - -This document lists all CLI v2 commands: diff --git a/content/cli-int.md b/content/cli-int.md index 168f27e05..d83b506b8 100644 --- a/content/cli-int.md +++ b/content/cli-int.md @@ -1,3 +1,5 @@ +# Interpolating Variables + !!! note Applies to CLI v2. @@ -7,7 +9,8 @@ It's typically necessary to separate passwords, certificates, S3 bucket names et Changing structure of a YAML document such as adding an S3 access configuration section is a bit more than just YAML document parameterization. Look into [operations files](cli-ops-files.md) for additional details. --- -## Variables {: #variables } + +## Variables {: #variables } Variables provide a way to define parameters for a YAML document. Each variable has a value, one or more reference locations and an optional type and generation options. @@ -126,7 +129,6 @@ A variable can define its type and generation options. !!! note Variables generated by the director into a config server (typically credhub) are not deleted upon deletion of the associated bosh deployment. This feature protects important credentials from being deleted upon accidental deletion of a deployment (similarly to persistent disks). It might therefore be necessary to directly periodically clean up orphaned credhub credentials. Bosh does not support such clean up. - ### `--vars-store` flag {: #vars-store } `--vars-store=path` flag provides a read write value source unlike all other variables flags that provide read only source. It is able to lazily generate and save (to a given file location) variable values based on their type and options. @@ -180,4 +182,3 @@ variables: !!!warning Variables can be referenced between deployments using their absolute path, however this is strongly discouraged and unsupported. BOSH provides a robust and preferred way to share properties between deployments, via [Links](links.md). - diff --git a/content/cli-ops-files.md b/content/cli-ops-files.md index 076a63a31..d9691ddd9 100644 --- a/content/cli-ops-files.md +++ b/content/cli-ops-files.md @@ -1,3 +1,5 @@ +# Creating Ops Files + It's usually necessary to apply an opinionated set of structural changes to a YAML document (manifest, cloud config, etc.) before submitting it to the CLI commands (`bosh create-env`, `bosh deploy`, etc.) for processing. Such changes could be an addition or removal of certain job properties, instance groups, changes to property values. !!! tip @@ -10,6 +12,7 @@ A single operation represents a single change. An operations file is a YAML docu Several CLI commands such as `create-env`, `deploy` and `interpolate` allow to provide operations files via `--ops-file` flag to be applied before processing the document. --- + ## Example {: #example } Following is an operations file (`replace-name.yml`) with a single operation that replaces value of top level key `name` with a string `other-cf`: @@ -40,6 +43,7 @@ bosh interpolate base.yml --ops-file replace-name.yml ``` --- + ## Path syntax {: #path } Each operation acts on a location within a YAML document. Path represents a location. It's important to note that path (location) does not represent what operation will be performed, just like lat & long do not represent what happens at a physical location. @@ -58,8 +62,8 @@ All paths follow these rules: - Paths always start at the root of the document with a `/` - String components typically refer to hash keys (ex: `/key1`) - - Strings ending with `?` refer to hash keys that may or may not exist - - "optionality" carries over to the items to the right + - Strings ending with `?` refer to hash keys that may or may not exist + - "optionality" carries over to the items to the right - Integer components refer to array indices (ex: `/0`, `/-1`) @@ -68,14 +72,15 @@ All paths follow these rules: - Array insertion could be affected via `:before` and `:after` (as of CLI v2.0.40+) - `-` component refers to an imaginary index after last array index (ex: `/-`) - - If there is an array of length 3 (`[0,1,2]`), then `-` would refer to 4th non-existent position + - If there is an array of length 3 (`[0,1,2]`), then `-` would refer to 4th non-existent position - `key=val` component matches hashes within an array (ex: `/key=val`) - - Values ending with `?` refer to array items that may or may not exist + - Values ending with `?` refer to array items that may or may not exist Path components without "optional" (`?`) annotation imply that referenced location must exist within a document. Operation will fail to be performed if that location is not found. "Optional" annotation can be used to signify indifference to the presense of referenced location, making it possible for operation either to ignore it (while removal) or create it lazily (while replacement). If a component in a path is annotated as optional, components following it will be considered optional implicitly. --- + ## Operations {: #ops } There are currently two types of operations: replace and remove. @@ -101,6 +106,7 @@ items: ``` --- + ### Hash Set `key` to `10`... @@ -336,16 +342,15 @@ items: - name: item8 ``` - ## Escaping The following characters can be escaped with special sequences... | Desired | Escaped | | ------- | ------- | -| `~` | `~0` | -| `/` | `~1` | -| `:` | `~7` | +| `~` | `~0` | +| `/` | `~1` | +| `:` | `~7` | For example, to remove a variable with a `name` of `/root_certificate`, you might do... diff --git a/content/cli-tunnel.md b/content/cli-tunnel.md index cc1500b72..9f81b7c79 100644 --- a/content/cli-tunnel.md +++ b/content/cli-tunnel.md @@ -1,3 +1,5 @@ +# Tunneling + !!! note Applies to CLI v2. @@ -10,7 +12,7 @@ Common use cases for tunnelling through a jumpbox VM include: The tunnel can be created by the CLI or established separately. -### Tunnel created by CLI +## Tunnel created by CLI ```shell # provide CLI with SSH credentials to create a tunnel via the environment variable @@ -20,7 +22,7 @@ bosh create-env bosh-deployment/bosh.yml ... bosh alias-env aws -e director-ip --ca-cert ... ``` -### Tunnel established separately +## Tunnel established separately ```shell # establish a tunnel and make it available on a local port diff --git a/content/cli-v2-diff.md b/content/cli-v2-diff.md index fbfa42a75..277685bb2 100644 --- a/content/cli-v2-diff.md +++ b/content/cli-v2-diff.md @@ -1,3 +1,5 @@ +# From the Ruby CLI + !!! note Applies to CLI v2 v2.0.13+. @@ -5,7 +7,7 @@ The BOSH CLI v2 differs from v1 in two main ways: it is stateless, and it hyphenates single commands. -Statelessness +### Statelessness The BOSH CLI v2 does not store values for a current environment or configuration. In v1, you set the environment by passing a Director endpoint to `bosh target` and set the deployment by passing a manifest @@ -14,30 +16,30 @@ file to `bosh deployment`. Then you could run `bosh deploy` with no arguments. In contrast, the BOSH CLI v2 is stateless. To specify a Director instance and deployment manifest to run a command over, you do one of the following: -* Pass the BOSH environment in with the `-e` flag and the deployment in with the `-d` flag, or -* Set the command shell environment variable `BOSH_ENVIRONMENT` to your Director endpoint or alias and set `BOSH_DEPLOYMENT` to your deployment name. You can also use `bosh alias-env` to create an alias for your BOSH environment configuration, to avoid having to reference the Director endpoint and credential information for every command. +- Pass the BOSH environment in with the `-e` flag and the deployment in with the `-d` flag, or +- Set the command shell environment variable `BOSH_ENVIRONMENT` to your Director endpoint or alias and set `BOSH_DEPLOYMENT` to your deployment name. You can also use `bosh alias-env` to create an alias for your BOSH environment configuration, to avoid having to reference the Director endpoint and credential information for every command. -Hyphenation +### Hyphenation The BOSH v2 CLI also hyphenates single commands that v1 represented as space-separated word pairs. For example, `bosh delete deployment` in v1 corresponds to `bosh delete-deployment` in v2. -| Before | After -|-----------------------------|----------------------------- -| bosh-init deploy | bosh create-env -| bosh-init delete | bosh delete-env -| bosh target | bosh alias-env my-env -e -| bosh status | bosh env -| bosh -t my-env ... | bosh -e my-env ... -| bosh -d manifest-path ... | bosh -d deployment-name ... [3] -| bosh deployment | n/a -| bosh deploy | bosh deploy -| bosh delete deployment | bosh delete-deployment -| bosh tasks --no-filter | bosh tasks -| bosh tasks recent 1000 | bosh tasks -r=1000 -| bosh download manifest dep | bosh manifest -| bosh vms my-dep | bosh instances -| bosh vms my-dep | bosh -d my-dep vms +| Before | After | +|-----------------------------|---------------------------------| +| bosh-init deploy | bosh create-env | +| bosh-init delete | bosh delete-env | +| bosh target | bosh alias-env my-env -e | +| bosh status | bosh env | +| bosh -t my-env ... | bosh -e my-env ... | +| bosh -d manifest-path ... | bosh -d deployment-name ... [3] | +| bosh deployment | n/a | +| bosh deploy | bosh deploy | +| bosh delete deployment | bosh delete-deployment | +| bosh tasks --no-filter | bosh tasks | +| bosh tasks recent 1000 | bosh tasks -r=1000 | +| bosh download manifest dep | bosh manifest | +| bosh vms my-dep | bosh instances | +| bosh vms my-dep | bosh -d my-dep vms | - Most commands require (`--environment`) `-e` and `--deployment` (`-d`) flags - `--deployment` (`-d`) flag accepts a deployment name instead of a manifest @@ -52,6 +54,7 @@ For example, `bosh delete deployment` in v1 corresponds to `bosh delete-deployme - Most of the output formatting have changed --- + ## Notable differences per command {: #cmd } - `bosh alias-env` and all commands diff --git a/content/cli-v2-install.md b/content/cli-v2-install.md index bf568c987..cac92b39d 100644 --- a/content/cli-v2-install.md +++ b/content/cli-v2-install.md @@ -2,12 +2,10 @@ The `bosh` CLI is the command line tool used for interacting with all things BOSH, from deployment operations to software release management. - ## Install Choose your preferred installation method below to get the latest version of `bosh`. - ### Using the binary directly To install the `bosh` binary directly: @@ -57,10 +55,10 @@ When you are using `bosh` to bootstrap BOSH or other standalone VMs, you will ne !!! tip If you will not be using `create-env` and `delete-env` commands, you can skip this section. - ### Ubuntu For Ubuntu Jammy (22.04), ensure the following packages are installed on your system: + ```shell sudo apt-get install -y build-essential zlib1g-dev ruby ruby-dev openssl libxslt1-dev libxml2-dev libssl-dev libreadline-dev libyaml-dev libsqlite3-dev sqlite3 ``` @@ -79,7 +77,6 @@ sudo apt-get install -y build-essential zlib1g-dev ruby ruby-dev openssl libxslt brew install openssl ``` - ### CentOS If you are running on CentOS, ensure the following packages are installed on your system: @@ -89,7 +86,6 @@ sudo yum install gcc gcc-c++ ruby ruby-devel mysql-devel postgresql-devel postgr gem install yajl-ruby ``` - ### Windows The `create-env` and `delete-env` commands are not yet supported on native Windows. Feel free to give it a try (and let us know if you have feedback), but we would recommend leveraging the Windows Subsystem for Linux if you need to run either command. @@ -98,5 +94,5 @@ The `create-env` and `delete-env` commands are not yet supported on native Windo You should be able to use `bosh` on other systems... we just don't know the exact packages to recommend. In general, use these recommendations (and send us a pull request to update this page once you figure it out!): - * compilation tools (often a `build-essential`-like package or "Development Tools"-like group) - * Ruby v2.4+ +- compilation tools (often a `build-essential`-like package or "Development Tools"-like group) +- Ruby v2.4+ diff --git a/content/cli-v2.md b/content/cli-v2.md index 20bdca164..ee95d84a9 100644 --- a/content/cli-v2.md +++ b/content/cli-v2.md @@ -1,3 +1,5 @@ +# Commands + !!! note Applies to CLI v3.0.1+. @@ -5,6 +7,7 @@ Installation of the BOSH CLI is required as a prerequisite, see [Installing the Release notes can be found [on Github](https://github.com/cloudfoundry/bosh-cli/releases). --- + ## Global BOSH CLI Application options These options are available to every command execution: @@ -143,6 +146,7 @@ See [Environments](cli-envs.md). ``` --- + ### Session {: #session-mgmt } #### Log-In {: #log-in } @@ -169,9 +173,11 @@ See [Environments](cli-envs.md). Logs out currently logged in user. --- + ### Director Environment {: #director-env} #### Environment {: #environment } + - `bosh [GLOBAL-CLI-OPTIONS] environment [--details]` (Alias: `env`) Shows Director information in the deployment environment. @@ -207,9 +213,11 @@ See [Environments](cli-envs.md). Succeeded ``` + The `Days left` column is has a visual shortcut that lists certificates with less than 30 full days of validity in _red_, and all others in _green_. --- + ### Stemcells {: #stemcell-mgmt } See [Uploading Stemcells](uploading-stemcells.md). @@ -255,7 +263,6 @@ See [Uploading Stemcells](uploading-stemcells.md). - `--sha1=DIGEST` SHA1 of the remote stemcell (is not used with local files) - `URL` Path to a local file or URL - ```shell bosh -e my-env us ~/Downloads/bosh-stemcell-621.74-warden-boshlite-ubuntu-xenial-go_agent.tgz bosh -e my-env us https://bosh.io/d/stemcells/bosh-warden-boshlite-ubuntu-xenial-go_agent?v=621.74 @@ -303,6 +310,7 @@ See [Uploading Stemcells](uploading-stemcells.md). ``` --- + ### Release creation {: #release-creation } #### Init-Release {: #init-release } @@ -408,6 +416,7 @@ See [Uploading Stemcells](uploading-stemcells.md). - `--dir=DIR` Release directory path if not current working directory (default: .) --- + ### Release blobs {: #blob-mgmt } See [Release Blobs](release-blobs.md) for a detailed workflow. @@ -490,6 +499,7 @@ See [Release Blobs](release-blobs.md) for a detailed workflow. ``` --- + ### Releases {: #release-mgmt } See [Uploading Releases](uploading-releases.md). @@ -675,6 +685,7 @@ See [Uploading Releases](uploading-releases.md). ``` --- + ### Configs {: #configs-mgmt } See [Configs](configs.md). @@ -779,6 +790,7 @@ See [Configs](configs.md). ``` --- + ### Cloud config {: #cloud-config-mgmt } See [Cloud config](cloud-config.md). @@ -811,6 +823,7 @@ See [Cloud config](cloud-config.md). ``` --- + ### Runtime config {: #runtime-config-mgmt } See [Runtime config](runtime-config.md). @@ -844,6 +857,7 @@ See [Runtime config](runtime-config.md). ``` --- + ### CPI config {: #cpi-config-mgmt } See [CPI config](cpi-config.md). @@ -887,6 +901,7 @@ bosh -e my-env config --type=deploy --name=default ``` --- + ### Deployments {: #deployment-mgmt } #### Deployments {: #deployments } @@ -990,7 +1005,7 @@ bosh -e my-env config --type=deploy --name=default ``` !!! note - As of CLI v7.5.0+ the deploy command supports the usage of global flags which are applied to every deploy command automatically. Please refer to [Deploy config](deploy-config.md) for more information. + As of CLI v7.5.0+ the deploy command supports the usage of global flags which are applied to every deploy command automatically. Please refer to [Deploy config](deploy-config.md) for more information. #### Delete-Deployment {: #delete-deployment } @@ -1061,6 +1076,7 @@ bosh -e my-env config --type=deploy --name=default bosh -e vbox -d cf recreate diego-cell/209c42e5-3c1a-432a-8445-ab8d7c9f69b0 --skip-drain bosh -e vbox -d cf recreate diego-cell --canaries=0 --max-in-flight=100% ``` + !!! warning In case of a **failed** deployment, running `bosh recreate` with the default behavior of `--converge` will converge to the last **successfully deployed state**, not the intended state of the failed deployment. See [Deployment Convergence](deployment-convergence.md). @@ -1203,6 +1219,7 @@ bosh -e my-env config --type=deploy --name=default List variables referenced by the deployment. --- + ### VMs {: #vm-mgmt } #### Vms {: #vms } @@ -1241,6 +1258,7 @@ bosh -e my-env config --type=deploy --name=default ``` --- + ### Disks {: #disk-mgmt } #### Disks {: #disks } @@ -1283,8 +1301,8 @@ bosh -e my-env config --type=deploy --name=default bosh -e vbox -d cf delete-disk vol-shw8f293f2f2 ``` - --- + ### SSH {: #ssh-mgmt } #### SSH {: #ssh } @@ -1359,6 +1377,7 @@ bosh -e my-env config --type=deploy --name=default ``` --- + ### PCAP {: #pcap-mgmt } #### PCAP {: #pcap } @@ -1402,6 +1421,7 @@ bosh -e my-env config --type=deploy --name=default ``` --- + ### Errands {: #errand-mgmt } #### Errands {: #errands } @@ -1475,6 +1495,7 @@ bosh -e my-env config --type=deploy --name=default ``` --- + ### Tasks {: #task-mgmt } #### Tasks {: #tasks } @@ -1483,7 +1504,7 @@ bosh -e my-env config --type=deploy --name=default Lists active and previously ran tasks. - - `-r`,` --recent=NUMBER` Show 30 recent tasks. Use '=' to specify the number of tasks to show + - `-r`,`--recent=NUMBER` Show 30 recent tasks. Use '=' to specify the number of tasks to show - `-a`, `--all` Include all task types (ssh, logs, vms, etc) ```shell @@ -1566,6 +1587,7 @@ bosh -e my-env config --type=deploy --name=default ``` --- + ### Snapshots {: #snapshot-mgmt } #### Snapshots {: #snapshots } @@ -1597,6 +1619,7 @@ bosh -e my-env config --type=deploy --name=default Deletes snapshots for an entire deployment. --- + ### Deployment recovery {: #deployment-recovery } #### Update-Resurrection {: #update-resurrection } @@ -1648,6 +1671,7 @@ bosh -e my-env config --type=deploy --name=default Lists current locks. --- + ### Network {: #network} #### Networks {: #networks } @@ -1673,6 +1697,7 @@ bosh -e my-env config --type=deploy --name=default ``` --- + ### Misc {: #misc } #### Clean-Up {: #clean-up } @@ -1689,7 +1714,6 @@ bosh -e my-env config --type=deploy --name=default **Note:** From BOSH v270.5.0, releases specified in runtime configs are considered 'in use' and won't be deleted by running `bosh clean-up [--all]`. - #### Help {: #help } - `bosh help` @@ -1770,7 +1794,6 @@ bosh -e my-env config --type=deploy --name=default - `--show-headers` `(-i)` show HTTP headers in the response - `PATH` URL path which can include query string - ```shell bosh curl /deployments bosh curl /links?deployment=my-dep diff --git a/content/cloud-config.md b/content/cloud-config.md index 2c9afedca..eac14adb2 100644 --- a/content/cloud-config.md +++ b/content/cloud-config.md @@ -1,3 +1,5 @@ +# Usage + !!! warning If you are using Director version between v241 and v256, once you opt into using cloud config all deployments must be converted to use new format. If you want to deploy both v1 and v2 manifests, update to Director v257+. @@ -6,6 +8,7 @@ Previously each deployment manifest specified IaaS and IaaS agnostic configurati The cloud config is a YAML file that defines IaaS specific configuration used by the Director and all deployments. It allows us to separate IaaS specific configuration into its own file and keep deployment manifests IaaS agnostic. --- + ## Updating and retrieving cloud config {: #update } To update cloud config on the Director use [`bosh update-cloud-config` command](cli-v2.md#update-cloud-config). @@ -50,6 +53,7 @@ Succeeded ``` --- + ## Example {: #example } ```yaml @@ -111,12 +115,13 @@ compilation: - [See vSphere CPI example](vsphere-cpi.md#cloud-config) --- + ## AZs Block {: #azs } **azs** [Array, required]: Specifies the AZs available to deployments. At least one should be specified. -* **name** [String, required]: Name of an AZ within the Director. -* **cloud_properties** [Hash, optional]: Describes any IaaS-specific properties needed to associated with AZ; for most IaaSes, some data here is actually required. See [CPI Specific `cloud_properties`](#azs-cloud-properties) below. Example: `availability_zone`. Default is `{}` (empty Hash). +- **name** [String, required]: Name of an AZ within the Director. +- **cloud_properties** [Hash, optional]: Describes any IaaS-specific properties needed to associated with AZ; for most IaaSes, some data here is actually required. See [CPI Specific `cloud_properties`](#azs-cloud-properties) below. Example: `availability_zone`. Default is `{}` (empty Hash). See [first class AZs](azs.md) for more details. @@ -141,6 +146,7 @@ azs: - [See vSphere CPI AZ cloud properties](vsphere-cpi.md#azs) --- + ## Networks Block {: #networks } **networks** [Array, required]: Each sub-block listed in the Networks block specifies a network configuration that jobs can reference. There are three different network types: `manual`, `dynamic`, and `vip`. At least one should be specified. @@ -156,12 +162,13 @@ See [networks](networks.md) for more details. - [See vSphere CPI network cloud properties](vsphere-cpi.md#networks) --- + ## VM Types Block {: #vm-types } **vm_types** [Array, required]: Specifies the [VM types](terminology.md#vm-type) available to deployments. At least one should be specified. -* **name** [String, required]: A unique name used to identify and reference the VM type -* **cloud_properties** [Hash, optional]: Describes any IaaS-specific properties needed to create VMs; for most IaaSes, some data here is actually required. See [CPI Specific `cloud_properties`](#vm-types-cloud-properties) below. Example: `instance_type: m3.medium`. Default is `{}` (empty Hash). +- **name** [String, required]: A unique name used to identify and reference the VM type +- **cloud_properties** [Hash, optional]: Describes any IaaS-specific properties needed to create VMs; for most IaaSes, some data here is actually required. See [CPI Specific `cloud_properties`](#vm-types-cloud-properties) below. Example: `instance_type: m3.medium`. Default is `{}` (empty Hash). Example: @@ -181,6 +188,7 @@ vm_types: - [See vSphere CPI VM types cloud properties](vsphere-cpi.md#resource-pools) --- + ## VM Extensions Block {: #vm-extensions } !!! note @@ -188,8 +196,8 @@ vm_types: **vm_extensions** [Array, optional]: Specifies the [VM extensions](terminology.md#vm-extension) available to deployments. -* **name** [String, required]: A unique name used to identify and reference the VM extension -* **cloud_properties** [Hash, optional]: Describes any IaaS-specific properties needed to configure VMs. Example: `elbs: [...]`. Default is `{}` (empty Hash). +- **name** [String, required]: A unique name used to identify and reference the VM extension +- **cloud_properties** [Hash, optional]: Describes any IaaS-specific properties needed to configure VMs. Example: `elbs: [...]`. Default is `{}` (empty Hash). Example: @@ -203,13 +211,14 @@ vm_extensions: Any IaaS specific configuration could be placed into a VM extension's `cloud_properties`. --- + ## Disk Types Block {: #disk-types } **disk_types** [Array, required]: Specifies the [disk types](terminology.md#disk-types) available to deployments. At least one should be specified. -* **name** [String, required]: A unique name used to identify and reference the disk type -* **disk_size** [Integer, required]: Specifies the disk size. `disk_size` must be a positive integer. BOSH creates a [persistent disk](persistent-disks.md) of that size in megabytes and attaches it to each job instance VM. -* **cloud_properties** [Hash, optional]: Describes any IaaS-specific properties needed to create disks. Examples: `type`, `iops`. Default is `{}` (empty Hash). +- **name** [String, required]: A unique name used to identify and reference the disk type +- **disk_size** [Integer, required]: Specifies the disk size. `disk_size` must be a positive integer. BOSH creates a [persistent disk](persistent-disks.md) of that size in megabytes and attaches it to each job instance VM. +- **cloud_properties** [Hash, optional]: Describes any IaaS-specific properties needed to create disks. Examples: `type`, `iops`. Default is `{}` (empty Hash). Example: @@ -230,22 +239,23 @@ disk_types: - [See vSphere CPI disk type cloud properties](vsphere-cpi.md#disk-pools) --- + ## Compilation Block {: #compilation } The Director creates compilation VMs for release compilation. The Director will compile each release on every necessary stemcell used in a deployment. A compilation definition allows to specify VM characteristics. **compilation** [Hash, required]: Properties of compilation VMs. -* **workers** [Integer, required]: The maximum number of compilation VMs. -* **az** [String, required]: Name of the AZ defined in AZs section to use for creating compilation VMs. -* **vm_type** [String, optional]: Name of the VM type defined in VM types section to use for creating compilation VMs. Alternatively, you can specify the `vm_resources`, or `cloud_properties` key. -* **orphan_workers** [Boolean, optional]: When enabled, BOSH will orphan compilation VMs after they finishing compiling packages for the VMs to be deleted asynchronously (instead of blocking the deployment). Default `false`. Available in bosh-release v267+. -* **vm_resources** [Hash, optional]: Specifies generic VM resources such as CPU, RAM and disk size that are automatically translated into correct VM cloud properties to determine VM size. VM size is determined on best effort basis as some IaaSes may not support exact size configuration. Currently some CPIs (Google and Azure) do not support this functionality. Available in bosh-release v264+. -* **vm_extensions** [Array, optional]: Names of the VM extensions defined in the VM extensions section to use for creating compilation VMs. -* **cloud_properties** [Hash, optional]: Describes any IaaS-specific properties needed to create VMs. Most IaaSes require this. Examples: `instance_type`, `availability_zone`. Default is `{}` (empty Hash). -* **network** [String, required]: References a valid network name defined in the Networks block. BOSH assigns network properties to compilation VMs according to the type and properties of the specified network. -* **reuse\_compilation\_vms** [Boolean, optional]: If `false`, BOSH creates a new compilation VM for each new package compilation and destroys the VM when compilation is complete. If `true`, compilation VMs are re-used when compiling packages. Defaults to `false`. -* **env** [Hash, optional]: Same as [`env` for instance groups](manifest-v2.md#instance-groups). +- **workers** [Integer, required]: The maximum number of compilation VMs. +- **az** [String, required]: Name of the AZ defined in AZs section to use for creating compilation VMs. +- **vm_type** [String, optional]: Name of the VM type defined in VM types section to use for creating compilation VMs. Alternatively, you can specify the `vm_resources`, or `cloud_properties` key. +- **orphan_workers** [Boolean, optional]: When enabled, BOSH will orphan compilation VMs after they finishing compiling packages for the VMs to be deleted asynchronously (instead of blocking the deployment). Default `false`. Available in bosh-release v267+. +- **vm_resources** [Hash, optional]: Specifies generic VM resources such as CPU, RAM and disk size that are automatically translated into correct VM cloud properties to determine VM size. VM size is determined on best effort basis as some IaaSes may not support exact size configuration. Currently some CPIs (Google and Azure) do not support this functionality. Available in bosh-release v264+. +- **vm_extensions** [Array, optional]: Names of the VM extensions defined in the VM extensions section to use for creating compilation VMs. +- **cloud_properties** [Hash, optional]: Describes any IaaS-specific properties needed to create VMs. Most IaaSes require this. Examples: `instance_type`, `availability_zone`. Default is `{}` (empty Hash). +- **network** [String, required]: References a valid network name defined in the Networks block. BOSH assigns network properties to compilation VMs according to the type and properties of the specified network. +- **reuse\_compilation\_vms** [Boolean, optional]: If `false`, BOSH creates a new compilation VM for each new package compilation and destroys the VM when compilation is complete. If `true`, compilation VMs are re-used when compiling packages. Defaults to `false`. +- **env** [Hash, optional]: Same as [`env` for instance groups](manifest-v2.md#instance-groups). Example: diff --git a/content/community.md b/content/community.md index 3454871d1..7f7293d43 100644 --- a/content/community.md +++ b/content/community.md @@ -2,7 +2,6 @@ BOSH is part of the open-source community, so you can easily find us in a few places... - ## Slack The [Cloud Foundry Slack](https://cloudfoundry.slack.com) is a great place to ask questions or discuss issues - especially if you are still trying to figure out what might actually be wrong. Contributors, operators, and developers of BOSH are typically hanging out in the [`#bosh`](https://cloudfoundry.slack.com/messages/C02HPPYQ2/) channel and happy to help answer questions. @@ -10,24 +9,20 @@ The [Cloud Foundry Slack](https://cloudfoundry.slack.com) is a great place to as !!! info "Slack Invitation" Slack requires members to be invited, so please [request an invite](https://slack.cloudfoundry.org/) from our bot first if you are new to the community. - ## GitHub BOSH is open source, so you can find most of the code in either the [`cloudfoundry`](https://github.com/cloudfoundry) or [`cloudfoundry-incubator`](https://github.com/cloudfoundry-incubator) GitHub organizations. If you are looking for a repository to start with, [`cloudfoundry/bosh`](https://github.com/cloudfoundry/bosh) is a great place to start. Since BOSH is a larger project, there are quite a few repositories for the various components. If you are not sure which repository is best to discuss an issue or make a change, please feel free to ask! [Slack](#slack) usually works best for that, but you can also just [create an issue](https://github.com/cloudfoundry/bosh/issues/new) if that's easier. - ## Roadmap We use [Pivotal Tracker](https://www.pivotaltracker.com/) for keeping track of what we're working on and prioritizing tasks. We have several projects, but our main project is [CF BOSH](https://www.pivotaltracker.com/n/projects/956238). For an aggregated view of all our projects, check out [bosh-stories](https://github.com/cloudfoundry/bosh-stories) to see a quick summary of our recent and upcoming work. - ## Mailing List If you prefer mailing lists, you can find us through the [`cf-bosh` mailing list](https://lists.cloudfoundry.org/g/cf-bosh). Although it is quieter than [Slack](#slack), the community is usually able to help answer questions raised on the list. We also occasionally use this for announcements or product discussion. - ## Additional Resources - * [Ultimate Guide to BOSH](https://ultimateguidetobosh.com/) - a great resource for learning more about BOSH concepts +* [Ultimate Guide to BOSH](https://ultimateguidetobosh.com/) - a great resource for learning more about BOSH concepts diff --git a/content/compiled-releases.md b/content/compiled-releases.md index 5cc6127af..85a22ba47 100644 --- a/content/compiled-releases.md +++ b/content/compiled-releases.md @@ -1,3 +1,5 @@ +# Creating Compiled Releases + !!! note This feature is available with bosh-release v210+. @@ -13,6 +15,7 @@ Typically release tarballs are distributed with source packages; however, there Any release can be exported as a compiled release by using the Director and [bosh export-release](cli-v2.md#export-release) command. --- + ## Using export-release command {: #export } To export a release: @@ -54,6 +57,7 @@ To export a release: 1. Optionally use `bosh inspect-release` command to view associated compiled packages on the Director. In our example: `bosh inspect-release uaa/45`. --- + ## Floating stemcells {: #floating } Compiled releases are built against a particular stemcell version. Director allows compiled releases to be installed on any minor version of the major stemcell version that the compiled release was exported against. `bosh create-env` command requires exact stemcell match unlike the Director. diff --git a/content/configs.md b/content/configs.md index db8d53eec..388970314 100644 --- a/content/configs.md +++ b/content/configs.md @@ -1,3 +1,5 @@ +# Generic Configs + !!! note Generic `configs` functionality is available with bosh-release v264+. @@ -9,10 +11,11 @@ Additionally, in some cases it may be useful to split cloud config and/or other ## Director Types {: #director-types } --There are four built-in types: `cloud`, `runtime`, `cpi` and `deploy`. You can interact with the Director config types just as you have been doing so far via the [`update-cloud-config`](cli-v2.md#cloud-config-mgmt), [`update-runtime-config`](cli-v2.md#runtime-config-mgmt) and [`update-cpi-config`](cli-v2.md#cpi-config-mgmt) CLI commands respectively. The `deploy` type can be managed using the generic [`update-config`](cli-v2.md#update-config) command with `--type=deploy` (see [Deploy config](deploy-config.md)). By using these commands you will only be able to interact with the `default` named config of the given type. This will be good enough in most cases but like in our example before if you need to create separate configs with different names, you need to use the [`update-config`](cli-v2.md#update-config) command. Keep in mind that if you use the [config commands](cli-v2.md#configs-mgmt) to interact with the built-in types, you still need to comply with the structure of the YAML file for each type. +- There are four built-in types: `cloud`, `runtime`, `cpi` and `deploy`. You can interact with the Director config types just as you have been doing so far via the [`update-cloud-config`](cli-v2.md#cloud-config-mgmt), [`update-runtime-config`](cli-v2.md#runtime-config-mgmt) and [`update-cpi-config`](cli-v2.md#cpi-config-mgmt) CLI commands respectively. The `deploy` type can be managed using the generic [`update-config`](cli-v2.md#update-config) command with `--type=deploy` (see [Deploy config](deploy-config.md)). By using these commands you will only be able to interact with the `default` named config of the given type. This will be good enough in most cases but like in our example before if you need to create separate configs with different names, you need to use the [`update-config`](cli-v2.md#update-config) command. Keep in mind that if you use the [config commands](cli-v2.md#configs-mgmt) to interact with the built-in types, you still need to comply with the structure of the YAML file for each type. +There are four built-in types: `cloud`, `runtime`, `cpi`, and `deploy`. You can interact with these via [`update-cloud-config`](cli-v2.md#cloud-config-mgmt), [`update-runtime-config`](cli-v2.md#runtime-config-mgmt), and [`update-cpi-config`](cli-v2.md#cpi-config-mgmt). The `deploy` type is managed via the generic [`update-config`](cli-v2.md#update-config) command with `--type=deploy` (see [Deploy config](deploy-config.md)). If you need consistent handling of named configs across types, use [`update-config`](cli-v2.md#update-config) with `--type` and `--name`. Keep in mind that when using [config commands](cli-v2.md#configs-mgmt) with built-in types, the YAML structure must match each type’s schema. --- + ## User defined Types {: #user-defined-types } In addition to the Director types an operator can set config of any other type using the [`update-config`](cli-v2.md#update-config) CLI command. The config file can be any file containing valid YAML. Root of the file must be a hash. @@ -20,6 +23,7 @@ In addition to the Director types an operator can set config of any other type u One of the use cases for providing such open ended functionality is to provide shared configuration API for supporting BOSH services instead of reimplementing something similar in each service. An upcoming example that will use this feature will be introduction of the `resurrection` config type that will allow operators to define custom resurrection rules, later read and interpreted by the Health Monitor. --- + ## Updating and retrieving a config {: #update } To add or update a config on the Director use the [`bosh update-config`](cli-v2.md#update-config) CLI command. @@ -65,6 +69,7 @@ Content configs: ``` --- + ## Listing configs {: #list } To list all configs use the [`bosh configs`](cli-v2.md#configs) CLI command. @@ -111,14 +116,17 @@ Succeeded ``` --- + ## Deleting configs {: #list } -To delete configs use the [`bosh delete-config`](cli-v2.md#delete-config) CLI command. +To delete configs use the [`bosh delete-config`](cli-v2.md#delete-config) CLI command. ```shell bosh delete-config 1 ``` + or + ```shell bosh delete-config --type=my-type --name=my-configs-name ``` diff --git a/content/cpi-api-rpc.md b/content/cpi-api-rpc.md index 12d731c20..074e02cf3 100644 --- a/content/cpi-api-rpc.md +++ b/content/cpi-api-rpc.md @@ -2,7 +2,6 @@ All CPIs are expected to implement a basic RPC interface through `STDIN`/`STDOUT`. - ## Invocation Since CPI is just an executable, following takes place for each CPI method call (where "caller" is often the director): @@ -16,17 +15,16 @@ Since CPI is just an executable, following takes place for each CPI method call For reference, there are two primary implementations of the "caller" which invoke the CPI... - * BOSH Director - invokes the CPI through typical `deploy`/stemcell commands, internally using its [`external_cpi.rb`](https://github.com/cloudfoundry/bosh/blob/master/src/bosh-director/lib/cloud/external_cpi.rb) wrapper. - * `bosh` CLI - invokes the CPI through `create-env`/`delete-env` commands, internally using its [`cpi_cmd_runner.go`](https://github.com/cloudfoundry/bosh-cli/blob/main/cloud/cpi_cmd_runner.go). - +- BOSH Director - invokes the CPI through typical `deploy`/stemcell commands, internally using its [`external_cpi.rb`](https://github.com/cloudfoundry/bosh/blob/master/src/bosh-director/lib/cloud/external_cpi.rb) wrapper. +- `bosh` CLI - invokes the CPI through `create-env`/`delete-env` commands, internally using its [`cpi_cmd_runner.go`](https://github.com/cloudfoundry/bosh-cli/blob/main/cloud/cpi_cmd_runner.go). ## API ### Request {: #request } -* `method` [String]: Name of the CPI method. Example: `create_vm`. -* `arguments` [Array]: Array of arguments that are specific to the CPI method. -* `context` [Hash]: Additional information provided as a context of this execution. +- `method` [String]: Name of the CPI method. Example: `create_vm`. +- `arguments` [Array]: Array of arguments that are specific to the CPI method. +- `context` [Hash]: Additional information provided as a context of this execution. An example request for [`delete_disk`](cpi-api-v2-method/delete-disk.md) might look like: @@ -39,11 +37,12 @@ An example request for [`delete_disk`](cpi-api-v2-method/delete-disk.md) might l ``` #### Context + The context contains additional information that may or may not be required in an individual CPI method. At the time of this writing (director version 270.11.0+), the following context is sent for each CPI call: -``` +```json { "director_uuid": "", "request_id": "", @@ -55,19 +54,19 @@ the following context is sent for each CPI call: } ``` + The CPI request ID is useful to trace a single invocation through the debug logs. The stemcell API version is sent if it is defined in the stemcell's manifest, for the stemcell this operation is relevant for (e.g. `create_vm`). Sometimes there is no stemcell involved in the operation, so this is undefined (e.g. `info`). The API version is used to determine eligibility for [registry-based property storage](cpi-api-v2-migration-guide.md#stemcell-changes-in-v2-of-the-api-contract). - ### Response {: #response } -* `result` [Null or simple values]: Single return value. It must be null if `error` is returned. -* `error` [Null or hash]: Occurred error. It must be null if `result` is returned. - * `type` [String]: Type of the error. - * `message` [String]: Description of the error. - * `ok_to_retry` [Boolean]: Indicates whether callee should try calling the method again without changing any of the arguments. -* `log` [String]: Additional information that may be useful for auditing, debugging and understanding what actions CPI took while executing a method. Typically includes info and debug logs, error backtraces. +- `result` [Null or simple values]: Single return value. It must be null if `error` is returned. +- `error` [Null or hash]: Occurred error. It must be null if `result` is returned. + - `type` [String]: Type of the error. + - `message` [String]: Description of the error. + - `ok_to_retry` [Boolean]: Indicates whether callee should try calling the method again without changing any of the arguments. +- `log` [String]: Additional information that may be useful for auditing, debugging and understanding what actions CPI took while executing a method. Typically includes info and debug logs, error backtraces. An example response to [`create_vm`](cpi-api-v2-method/create-vm.md) might look like: @@ -93,33 +92,32 @@ An example error response to [`create_vm`](cpi-api-v2-method/create-vm.md) might } ``` - ## Methods - * [info](cpi-api-v2-method/info.md) - * Stemcells - * [create_stemcell](cpi-api-v2-method/create-stemcell.md) - * [delete_stemcell](cpi-api-v2-method/delete-stemcell.md) - * VM Management - * [create_vm](cpi-api-v2-method/create-vm.md) - * [delete_vm](cpi-api-v2-method/delete-vm.md) - * [has_vm](cpi-api-v2-method/has-vm.md) - * [reboot_vm](cpi-api-v2-method/reboot-vm.md) - * [set_vm_metadata](cpi-api-v2-method/set-vm-metadata.md) - * [calculate_vm_cloud_properties](cpi-api-v2-method/calculate-vm-cloud-properties.md) - * Disk Management - * [create_disk](cpi-api-v2-method/create-disk.md) - * [delete_disk](cpi-api-v2-method/delete-disk.md) - * [resize_disk](cpi-api-v2-method/resize-disk.md) - * [update_disk](cpi-api-v2-method/update-disk.md) - * [has_disk](cpi-api-v2-method/has-disk.md) - * [attach_disk](cpi-api-v2-method/attach-disk.md) - * [detach_disk](cpi-api-v2-method/detach-disk.md) - * [set_disk_metadata](cpi-api-v2-method/set-disk-metadata.md) - * [get_disks](cpi-api-v2-method/get-disks.md) - * Snapshot Management - * [snapshot_disk](cpi-api-v2-method/snapshot-disk.md) - * [delete_snapshot](cpi-api-v2-method/delete-snapshot.md) - * Deprecated v1 methods - * [configure_networks](cpi-api-v1-method/configure-networks.md) - * [current_vm_id](cpi-api-v1-method/current-vm-id.md) +- [info](cpi-api-v2-method/info.md) +- Stemcells + - [create_stemcell](cpi-api-v2-method/create-stemcell.md) + - [delete_stemcell](cpi-api-v2-method/delete-stemcell.md) +- VM Management + - [create_vm](cpi-api-v2-method/create-vm.md) + - [delete_vm](cpi-api-v2-method/delete-vm.md) + - [has_vm](cpi-api-v2-method/has-vm.md) + - [reboot_vm](cpi-api-v2-method/reboot-vm.md) + - [set_vm_metadata](cpi-api-v2-method/set-vm-metadata.md) + - [calculate_vm_cloud_properties](cpi-api-v2-method/calculate-vm-cloud-properties.md) +- Disk Management + - [create_disk](cpi-api-v2-method/create-disk.md) + - [delete_disk](cpi-api-v2-method/delete-disk.md) + - [resize_disk](cpi-api-v2-method/resize-disk.md) + - [update_disk](cpi-api-v2-method/update-disk.md) + - [has_disk](cpi-api-v2-method/has-disk.md) + - [attach_disk](cpi-api-v2-method/attach-disk.md) + - [detach_disk](cpi-api-v2-method/detach-disk.md) + - [set_disk_metadata](cpi-api-v2-method/set-disk-metadata.md) + - [get_disks](cpi-api-v2-method/get-disks.md) + - Snapshot Management + - [snapshot_disk](cpi-api-v2-method/snapshot-disk.md) + - [delete_snapshot](cpi-api-v2-method/delete-snapshot.md) +- Deprecated v1 methods + - [configure_networks](cpi-api-v1-method/configure-networks.md) + - [current_vm_id](cpi-api-v1-method/current-vm-id.md) diff --git a/content/cpi-api-v1-method/attach-disk.md b/content/cpi-api-v1-method/attach-disk.md index 39a0cab7d..74d9ecbfb 100644 --- a/content/cpi-api-v1-method/attach-disk.md +++ b/content/cpi-api-v1-method/attach-disk.md @@ -6,59 +6,46 @@ Typically each VM will have one disk attached at a time to store persistent data Agent settings should have been updated with necessary information about given disk. - ## Arguments - * `vm_cid` [String]: Cloud ID of the VM. - * `disk_cid` [String]: Cloud ID of the disk. - +- `vm_cid` [String]: Cloud ID of the VM. +- `disk_cid` [String]: Cloud ID of the disk. ## Result No return value - ## Agent settings For the Agent to eventually format, partition and mount attached disk, it needs to identify the disk attachment from inside the OS. The Agent can currently identify attached disk based on either device path, disk's ID, or SCSI volume ID. For example settings below show that CPI attached a disk `vol-7447851` at `/dev/sdd`: ```json { - "agent_id": "4149ba0f-38d9-4485-476f-1581be36f290", - - "vm": { "name": "i-347844" }, - - "networks": { ... }, - - "disks": { - "system": "/dev/sda", - "ephemeral": "/dev/sdb", - "persistent": { - "vol-3475945": { "volume_id": "3" }, - "vol-7447851": { "path": "/dev/sdd" }, - } - }, - - "mbus": "https://mbus:mbus-password@0.0.0.0:6868", - - "ntp": [ ... ], - - "blobstore": { ... }, - - "env": {}, + "agent_id": "4149ba0f-38d9-4485-476f-1581be36f290", + "vm": { "name": "i-347844" }, + "networks": { ... }, + "disks": { + "system": "/dev/sda", + "ephemeral": "/dev/sdb", + "persistent": { + "vol-3475945": { "volume_id": "3" }, + "vol-7447851": { "path": "/dev/sdd" }, + } + }, + "mbus": "https://mbus:mbus-password@0.0.0.0:6868", + "ntp": [ ... ], + "blobstore": { ... }, + "env": {}, } ``` - ## Examples - ### Implementations - * [cloudfoundry/bosh-warden-cpi-release](https://github.com/cloudfoundry/bosh-warden-cpi-release/blob/master/src/bosh-warden-cpi/action/attach_disk.go) - +- [cloudfoundry/bosh-warden-cpi-release](https://github.com/cloudfoundry/bosh-warden-cpi-release/blob/master/src/bosh-warden-cpi/action/attach_disk.go) ## Related - * [create_disk](../cpi-api-v2-method/create-disk.md) - * [detach_disk](../cpi-api-v2-method/detach-disk.md) +- [create_disk](../cpi-api-v2-method/create-disk.md) +- [detach_disk](../cpi-api-v2-method/detach-disk.md) diff --git a/content/cpi-api-v1-method/configure-networks.md b/content/cpi-api-v1-method/configure-networks.md index 0b0f47d4a..98b4a133d 100644 --- a/content/cpi-api-v1-method/configure-networks.md +++ b/content/cpi-api-v1-method/configure-networks.md @@ -4,18 +4,15 @@ The recommended implementation is to raise `Bosh::Clouds::NotSupported` error. T After the Director received NotSupported error, it will delete the VM (via `delete_vm`) and create a new VM with desired network configuration (via `create_vm`). - ## Arguments - * `vm_cid` [String]: Cloud ID of the VM to modify; returned from `create_vm`. - * `networks` [Hash]: Network hashes that specify networks VM must be configured. - +- `vm_cid` [String]: Cloud ID of the VM to modify; returned from `create_vm`. +- `networks` [Hash]: Network hashes that specify networks VM must be configured. ## Result No return value - ## Examples ```json diff --git a/content/cpi-api-v1-method/create-vm.md b/content/cpi-api-v1-method/create-vm.md index 7af55af2f..529df330a 100644 --- a/content/cpi-api-v1-method/create-vm.md +++ b/content/cpi-api-v1-method/create-vm.md @@ -6,24 +6,21 @@ Waiting for the VM to finish booting is not required because the Director waits Make sure to properly delete created resources if VM cannot be successfully created. - ## Arguments -* `agent_id` [String]: ID selected by the Director for the VM's agent. -* `stemcell_cid` [String]: Cloud ID of the stemcell to use as a base image for new VM. -* `cloud_properties` [Hash]: Cloud properties hash specified in the deployment manifest under VM's resource pool. -* `networks` [Hash]: Networks hash that specifies which VM networks must be configured. -* `disk_cids` [Array of strings] Array of disk cloud IDs for each disk that created VM will most _likely_ be attached; they could be used to optimize VM placement so that disks are located nearby. -* `environment` [Hash]: Resource pool's env hash specified in deployment manifest. Additionally, the director will append the following guaranteed values: - * `bosh` [Hash]: A collection of properties used by the BOSH Agent, and optionally the CPI. - * `group` [String]: A description of the requested VM in the format `--`. - * `groups` [Array]: A collection of descriptions for the requested VM, combining `director-name`, `deployment-name` and `job-name` in a range of strings separated by a `-`. - +- `agent_id` [String]: ID selected by the Director for the VM's agent. +- `stemcell_cid` [String]: Cloud ID of the stemcell to use as a base image for new VM. +- `cloud_properties` [Hash]: Cloud properties hash specified in the deployment manifest under VM's resource pool. +- `networks` [Hash]: Networks hash that specifies which VM networks must be configured. +- `disk_cids` [Array of strings] Array of disk cloud IDs for each disk that created VM will most _likely_ be attached; they could be used to optimize VM placement so that disks are located nearby. +- `environment` [Hash]: Resource pool's env hash specified in deployment manifest. Additionally, the director will append the following guaranteed values: + - `bosh` [Hash]: A collection of properties used by the BOSH Agent, and optionally the CPI. + - `group` [String]: A description of the requested VM in the format `--`. + - `groups` [Array]: A collection of descriptions for the requested VM, combining `director-name`, `deployment-name` and `job-name` in a range of strings separated by a `-`. ## Result - * `vm_cid` [String]: Cloud ID of the created VM. - +- `vm_cid` [String]: Cloud ID of the created VM. ## Agent Settings @@ -72,10 +69,8 @@ For the Agent to successfully start on the created VM, several bootstrapping set See [Agent Configuration](../vm-config.md#agent) for an overview of the Agent configuration file locations. - ## Examples - ### API Request ```json @@ -115,12 +110,10 @@ See [Agent Configuration](../vm-config.md#agent) for an overview of the Agent co ] ``` - ### Implementations - * [cloudfoundry/bosh-warden-cpi-release](https://github.com/cloudfoundry/bosh-warden-cpi-release/blob/master/src/bosh-warden-cpi/action/create_vm.go) - +- [cloudfoundry/bosh-warden-cpi-release](https://github.com/cloudfoundry/bosh-warden-cpi-release/blob/master/src/bosh-warden-cpi/action/create_vm.go) ## Related - * [delete_vm](../cpi-api-v2-method/delete-vm.md) +- [delete_vm](../cpi-api-v2-method/delete-vm.md) diff --git a/content/cpi-api-v1-method/current-vm-id.md b/content/cpi-api-v1-method/current-vm-id.md index b89423ff0..89cb5e4e3 100644 --- a/content/cpi-api-v1-method/current-vm-id.md +++ b/content/cpi-api-v1-method/current-vm-id.md @@ -5,12 +5,10 @@ Determines cloud ID of the VM executing the CPI code. Currently used in combinat !!! note Do not implement; this method will be deprecated and removed. - ## Arguments No arguments - ## Returned - * `vm_cid` [String]: Cloud ID of the VM. +- `vm_cid` [String]: Cloud ID of the VM. diff --git a/content/cpi-api-v1-method/delete-vm.md b/content/cpi-api-v1-method/delete-vm.md index d0b38323e..1da1e81a3 100644 --- a/content/cpi-api-v1-method/delete-vm.md +++ b/content/cpi-api-v1-method/delete-vm.md @@ -6,24 +6,20 @@ This method will be called while the VM still has persistent disks attached. It' To avoid losing track of VMs, make sure to raise an error if VM deletion is not absolutely certain. - ## Arguments - **vm_cid** [String]: Cloud ID of the VM to delete; returned from `create_vm`. - ## Result No return value - ## Examples ### Implementations - * [bosh-warden-cpi-release](https://github.com/cloudfoundry/bosh-warden-cpi-release/blob/master/src/bosh-warden-cpi/action/delete_vm.go) - +- [bosh-warden-cpi-release](https://github.com/cloudfoundry/bosh-warden-cpi-release/blob/master/src/bosh-warden-cpi/action/delete_vm.go) ## Related - * [create_vm](../cpi-api-v2-method/create-vm.md) +- [create_vm](../cpi-api-v2-method/create-vm.md) diff --git a/content/cpi-api-v1-method/detach-disk.md b/content/cpi-api-v1-method/detach-disk.md index cbff7707d..83401275c 100644 --- a/content/cpi-api-v1-method/detach-disk.md +++ b/content/cpi-api-v1-method/detach-disk.md @@ -6,27 +6,22 @@ If the persistent disk is attached to a VM that will be deleted, it's more likel Agent settings should have been updated to remove information about given disk. - ## Arguments - * `vm_cid` [String]: Cloud ID of the VM. - * `disk_cid` [String]: Cloud ID of the disk. - +- `vm_cid` [String]: Cloud ID of the VM. +- `disk_cid` [String]: Cloud ID of the disk. ## Result No return value - ## Examples - ### Implementations - * [cloudfoundry/bosh-warden-cpi-release](https://github.com/cloudfoundry/bosh-warden-cpi-release/blob/master/src/bosh-warden-cpi/action/detach_disk.go) - +- [cloudfoundry/bosh-warden-cpi-release](https://github.com/cloudfoundry/bosh-warden-cpi-release/blob/master/src/bosh-warden-cpi/action/detach_disk.go) ## Related - * [attach_disk](../cpi-api-v2-method/attach-disk.md) - * [delete_disk](../cpi-api-v2-method/delete-disk.md) +- [attach_disk](../cpi-api-v2-method/attach-disk.md) +- [delete_disk](../cpi-api-v2-method/delete-disk.md) diff --git a/content/cpi-api-v1-method/info.md b/content/cpi-api-v1-method/info.md index f4d87bb70..ac4611a74 100644 --- a/content/cpi-api-v1-method/info.md +++ b/content/cpi-api-v1-method/info.md @@ -2,13 +2,11 @@ Returns information about the CPI to help the Director to make decisions on which CPI to call for certain operations in a multi CPI scenario. - ## Arguments No arguments - ## Result - * [Hash]: Information about the CPI. - * `stemcell_formats` [Array of strings]: Stemcell formats supported by the CPI. Currently used in combination with `create_stemcell` by the Director to determine which CPI to call when uploading a stemcell. +- [Hash]: Information about the CPI. + - `stemcell_formats` [Array of strings]: Stemcell formats supported by the CPI. Currently used in combination with `create_stemcell` by the Director to determine which CPI to call when uploading a stemcell. diff --git a/content/cpi-api-v1.md b/content/cpi-api-v1.md index dec054ea4..4702b0050 100644 --- a/content/cpi-api-v1.md +++ b/content/cpi-api-v1.md @@ -5,5 +5,4 @@ See [Migration guide](cpi-api-v2-migration-guide.md). -Methods in the Version 1 section of this reference include only those methods which -differ from, or are not included in, Version 2. \ No newline at end of file +Methods in the Version 1 section of this reference include only those methods which differ from, or are not included in, Version 2. diff --git a/content/cpi-api-v2-method/attach-disk.md b/content/cpi-api-v2-method/attach-disk.md index 81d093ee5..05346e7c8 100644 --- a/content/cpi-api-v2-method/attach-disk.md +++ b/content/cpi-api-v2-method/attach-disk.md @@ -4,20 +4,18 @@ Attaches a given disk to a given VM. ## Arguments - * `vm_cid` [String]: Cloud ID of the VM. - * `disk_cid` [String]: Cloud ID of the disk. - +- `vm_cid` [String]: Cloud ID of the VM. +- `disk_cid` [String]: Cloud ID of the disk. ## Result - * `disk_hints` [Hash or String]: Disks that are associated with the VM +- `disk_hints` [Hash or String]: Disks that are associated with the VM The `disk_hints` vary between each IaaS. The `disk_hints` describe the physical attach point of the disk. The Agent is updated with a mapping of volume ID to attach point. For example, the AWS implementation of the CPI simply returns a string representing the device block id: `"/dev/sdd"` - ## Agent settings For the Agent to eventually format, partition and mount the newly attached disk, it needs to identify the disk attachment from inside the OS. The Agent can currently identify attached disks based on either their device path, disk's ID, or SCSI volume ID. For example, the sample settings below show that the CPI attached a disk `vol-7447851` at `/dev/sdd`: @@ -28,12 +26,12 @@ For the Agent to eventually format, partition and mount the newly attached disk, "vm": { "name": "i-347844" }, "networks": { ... }, "disks": { - "system": "/dev/sda", - "ephemeral": "/dev/sdb", - "persistent": { - "vol-3475945": { "volume_id": "3" }, - "vol-7447851": { "path": "/dev/sdd" }, - } + "system": "/dev/sda", + "ephemeral": "/dev/sdb", + "persistent": { + "vol-3475945": { "volume_id": "3" }, + "vol-7447851": { "path": "/dev/sdd" }, + } }, "mbus": "https://mbus:mbus-password@0.0.0.0:6868", "ntp": [ ... ], @@ -54,7 +52,7 @@ For the Agent to eventually format, partition and mount the newly attached disk, "vol-01aad1d6b3149cca1" ], "context": { - "director_uuid": "", + "director_uuid": "", "request_id": "", "vm": { "stemcell": { @@ -79,14 +77,12 @@ For the Agent to eventually format, partition and mount the newly attached disk, See [CPI API V2](../cpi-api-v2.md) and [CPI V2 Migration Guide](../cpi-api-v2-migration-guide.md) for more details about `api_version` for stemcell and CPI within the `context` portion of the request. - ### Implementations - * [cloudfoundry/bosh-warden-cpi-release](https://github.com/cloudfoundry/bosh-warden-cpi-release/blob/master/src/bosh-warden-cpi/action/attach_disk.go) - +- [cloudfoundry/bosh-warden-cpi-release](https://github.com/cloudfoundry/bosh-warden-cpi-release/blob/master/src/bosh-warden-cpi/action/attach_disk.go) ## Related - * [attach_disk V1](../cpi-api-v1-method/attach-disk.md) - * [create_disk](create-disk.md) - * [detach_disk](detach-disk.md) +- [attach_disk V1](../cpi-api-v1-method/attach-disk.md) +- [create_disk](create-disk.md) +- [detach_disk](detach-disk.md) diff --git a/content/cpi-api-v2-method/calculate-vm-cloud-properties.md b/content/cpi-api-v2-method/calculate-vm-cloud-properties.md index 1ab3fc11d..3df4dc661 100644 --- a/content/cpi-api-v2-method/calculate-vm-cloud-properties.md +++ b/content/cpi-api-v2-method/calculate-vm-cloud-properties.md @@ -15,23 +15,19 @@ The `cloud_properties` returned are IaaS-specific. For example, when querying th If a parameter is set to a value greater than what is available (e.g. 1024 CPUs), an error is raised. - ## Arguments - * `desired_instance_size` [Hash]: Parameters of the desired size of the VM consisting of the following keys: - * `cpu` [Integer]: Number of virtual cores desired - * `ram` [Integer]: Amount of RAM, in MiB (i.e. `4096` for 4 GiB) - * `ephemeral_disk_size` [Integer]: Size of ephemeral disk, in MB - +- `desired_instance_size` [Hash]: Parameters of the desired size of the VM consisting of the following keys: + - `cpu` [Integer]: Number of virtual cores desired + - `ram` [Integer]: Amount of RAM, in MiB (i.e. `4096` for 4 GiB) + - `ephemeral_disk_size` [Integer]: Size of ephemeral disk, in MB ## Result - * `cloud_properties` [Hash]: an IaaS-specific set of cloud properties that define the size of the VM. - +- `cloud_properties` [Hash]: an IaaS-specific set of cloud properties that define the size of the VM. ## Examples - ### API Request ```json @@ -42,7 +38,6 @@ If a parameter is set to a value greater than what is available (e.g. 1024 CPUs) } ``` - ## Related - * [create_vm](create-vm.md) +- [create_vm](create-vm.md) diff --git a/content/cpi-api-v2-method/create-disk.md b/content/cpi-api-v2-method/create-disk.md index cf2803274..9a40e37e0 100644 --- a/content/cpi-api-v2-method/create-disk.md +++ b/content/cpi-api-v2-method/create-disk.md @@ -2,22 +2,18 @@ Creates disk with specific size. Disk does not belong to any given VM. - ## Arguments - * `size` [Integer]: Size of the disk in MiB. - * `cloud_properties` [Hash]: Cloud properties hash specified in the deployment manifest under the disk pool. - * `vm_cid` [String]: Cloud ID of the VM created disk will most _likely_ be attached; it could be used to .optimize disk placement so that disk is located near the VM. - +- `size` [Integer]: Size of the disk in MiB. +- `cloud_properties` [Hash]: Cloud properties hash specified in the deployment manifest under the disk pool. +- `vm_cid` [String]: Cloud ID of the VM created disk will most _likely_ be attached; it could be used to .optimize disk placement so that disk is located near the VM. ## Returned - * `disk_cid` [String]: Cloud ID of the created disk. - +- `disk_cid` [String]: Cloud ID of the created disk. ## Examples - ### API Request ```json @@ -33,10 +29,9 @@ Creates disk with specific size. Disk does not belong to any given VM. ### Implementations - * [cloudfoundry/bosh-warden-cpi-release](https://github.com/cloudfoundry/bosh-warden-cpi-release/blob/master/src/bosh-warden-cpi/action/create_disk.go) - +- [cloudfoundry/bosh-warden-cpi-release](https://github.com/cloudfoundry/bosh-warden-cpi-release/blob/master/src/bosh-warden-cpi/action/create_disk.go) ## Related - * [attach_disk](attach-disk.md) - * [delete_disk](delete-disk.md) +- [attach_disk](attach-disk.md) +- [delete_disk](delete-disk.md) diff --git a/content/cpi-api-v2-method/create-network.md b/content/cpi-api-v2-method/create-network.md index b9e355da7..b82979f60 100644 --- a/content/cpi-api-v2-method/create-network.md +++ b/content/cpi-api-v2-method/create-network.md @@ -4,9 +4,9 @@ Creates a network that will be used to place VMs on. ## Arguments -Properties required for creating the network. It may contain `range` and `gateway` keys. A `cloud_properties` is required to provide information specific to the CPI and target IaaS. +Properties required for creating the network. It may contain `range` and `gateway` keys. A `cloud_properties` is required to provide information specific to the CPI and target IaaS. -``` +```yaml { type: String (required) cloud_properties: Hash (required) @@ -18,8 +18,7 @@ Properties required for creating the network. It may contain `range` and `gatewa ## Result -* Array with the following format: `[network_id (string), addresses (hash), cloud properties (hash)]` - +- Array with the following format: `[network_id (string), addresses (hash), cloud properties (hash)]` ## Examples @@ -74,12 +73,10 @@ Properties required for creating the network. It may contain `range` and `gatewa } ``` - ### Implementations - * [cloudfoundry/bosh-vsphere-cpi-release](https://github.com/cloudfoundry/bosh-vsphere-cpi-release/blob/dfe878579cbab768af07a12bb5543cd016cbb762/src/vsphere_cpi/lib/cloud/vsphere/cloud.rb#L683) - +- [cloudfoundry/bosh-vsphere-cpi-release](https://github.com/cloudfoundry/bosh-vsphere-cpi-release/blob/dfe878579cbab768af07a12bb5543cd016cbb762/src/vsphere_cpi/lib/cloud/vsphere/cloud.rb#L683) ## Related - * [delete_network](delete-network.md) +- [delete_network](delete-network.md) diff --git a/content/cpi-api-v2-method/create-stemcell.md b/content/cpi-api-v2-method/create-stemcell.md index d9e8ee210..3cb718ffb 100644 --- a/content/cpi-api-v2-method/create-stemcell.md +++ b/content/cpi-api-v2-method/create-stemcell.md @@ -4,48 +4,43 @@ Creates a reusable VM image in the IaaS from the [stemcell](../stemcell.md) imag See [Stemcell Building](../build-stemcell.md) for more details. - ## Arguments - * `image_path` [String]: Path to the stemcell image extracted from the stemcell tarball on a local filesystem. - * `cloud_properties` [Hash]: Cloud properties hash extracted from the stemcell tarball. - +- `image_path` [String]: Path to the stemcell image extracted from the stemcell tarball on a local filesystem. +- `cloud_properties` [Hash]: Cloud properties hash extracted from the stemcell tarball. ## Result - * `stemcell_cid` [String]: Cloud ID of the created stemcell (e.g. stemcells in AWS CPI are made into AMIs so cid would be `ami-83fdflf`) - +- `stemcell_cid` [String]: Cloud ID of the created stemcell (e.g. stemcells in AWS CPI are made into AMIs so cid would be `ami-83fdflf`) ## Example - ### API Request ```json [ - "/tmp/extracted-stemcell-348754vdsn87fr/image", - { - "name": "bosh-openstack-esxi-ubuntu-xenial-go_agent", - "version": "621.74", - "infrastructure": "openstack", - "hypervisor": "esxi", - "disk": 3072, - "disk_format": "ovf", - "container_format": "bare", - "os_type": "linux", - "os_distro": "ubuntu", - "architecture": "x86_64", - "auto_disk_config": true - } + "/tmp/extracted-stemcell-348754vdsn87fr/image", + { + "name": "bosh-openstack-esxi-ubuntu-xenial-go_agent", + "version": "621.74", + "infrastructure": "openstack", + "hypervisor": "esxi", + "disk": 3072, + "disk_format": "ovf", + "container_format": "bare", + "os_type": "linux", + "os_distro": "ubuntu", + "architecture": "x86_64", + "auto_disk_config": true + } ] ``` ### Implementations - * [cloudfoundry/bosh-warden-cpi-release](https://github.com/cloudfoundry/bosh-warden-cpi-release/blob/master/src/bosh-warden-cpi/action/create_stemcell.go) - +- [cloudfoundry/bosh-warden-cpi-release](https://github.com/cloudfoundry/bosh-warden-cpi-release/blob/master/src/bosh-warden-cpi/action/create_stemcell.go) ## Related - * [delete_stemcell](delete-stemcell.md) - * [create_vm](create-vm.md) +- [delete_stemcell](delete-stemcell.md) +- [create_vm](create-vm.md) diff --git a/content/cpi-api-v2-method/create-vm.md b/content/cpi-api-v2-method/create-vm.md index 368786175..cb42a41e0 100644 --- a/content/cpi-api-v2-method/create-vm.md +++ b/content/cpi-api-v2-method/create-vm.md @@ -14,27 +14,24 @@ As of V2, created VMs can be tagged at creation time. Please note that the also the [`set_vm_metadata` CPI method](set-vm-metadata.md) for more info about default tags. - - ## Arguments - * `agent_id` [String]: ID selected by the Director for the VM's agent. - * `stemcell_cid` [String]: Cloud ID of the stemcell to use as a base image for new VM. - * `cloud_properties` [Hash]: Cloud properties hash specified in the deployment manifest under the VM's resource pool. - * `networks` [Hash]: Networks hash that specifies which VM networks must be configured. - * `disk_cids` [Array of strings] Array of disk cloud IDs for the disks that the created VM will most _likely_ attach. The disk cloud IDs could be used to optimize VM placement so that disks are located nearby. - * `environment` [Hash]: Resource pool's env hash specified in the deployment manifest, including initial properties added by the BOSH director as shown below. The CPI adds it to the VM's `user data` which is then used by the agent. Additionally, the director will append the following guaranteed values: - * `bosh` [Hash]: A collection of properties used by the BOSH Agent, and optionally the CPI. - * `group` [String]: A description of the requested VM in the format `--`. - * `groups` [Array]: A collection of descriptions for the requested VM, combining `director-name`, `deployment-name` and `job-name` in a range of strings separated by a `-`. - * `tags` [Hash]: Tags from the top-level `tags` in the deployment manifest. Some security policies on some IaaSes require tags during VM creation. e.g. `{'tag-name': 'tag-value'}`. As of director v270.7.0+. - +- `agent_id` [String]: ID selected by the Director for the VM's agent. +- `stemcell_cid` [String]: Cloud ID of the stemcell to use as a base image for new VM. +- `cloud_properties` [Hash]: Cloud properties hash specified in the deployment manifest under the VM's resource pool. +- `networks` [Hash]: Networks hash that specifies which VM networks must be configured. +- `disk_cids` [Array of strings] Array of disk cloud IDs for the disks that the created VM will most _likely_ attach. The disk cloud IDs could be used to optimize VM placement so that disks are located nearby. +- `environment` [Hash]: Resource pool's env hash specified in the deployment manifest, including initial properties added by the BOSH director as shown below. The CPI adds it to the VM's `user data` which is then used by the agent. Additionally, the director will append the following guaranteed values: + - `bosh` [Hash]: A collection of properties used by the BOSH Agent, and optionally the CPI. + - `group` [String]: A description of the requested VM in the format `--`. + - `groups` [Array]: A collection of descriptions for the requested VM, combining `director-name`, `deployment-name` and `job-name` in a range of strings separated by a `-`. + - `tags` [Hash]: Tags from the top-level `tags` in the deployment manifest. Some security policies on some IaaSes require tags during VM creation. e.g. `{'tag-name': 'tag-value'}`. As of director v270.7.0+. ## Result -* Array of results - * `vm_cid` [String]: Cloud ID of the created VM. - * `networks` [Hash]: Networks associated with the VM. +- Array of results + - `vm_cid` [String]: Cloud ID of the created VM. + - `networks` [Hash]: Networks associated with the VM. ## Agent Settings @@ -81,10 +78,8 @@ Most CPIs choose to communicate with the default Agent. Hence, the communication See [Agent Configuration](../vm-config.md#agent) for an overview of the Agent configuration file locations. - ## Examples - ### API Request ```json @@ -167,10 +162,9 @@ Response: ### Implementations - * [cloudfoundry/bosh-aws-cpi-release](https://github.com/cloudfoundry/bosh-aws-cpi-release/blob/00e11f480847a4e88533f1e95b7c626a213d780b/src/bosh_aws_cpi/lib/cloud/aws/cloud_v2.rb#L41) - +- [cloudfoundry/bosh-aws-cpi-release](https://github.com/cloudfoundry/bosh-aws-cpi-release/blob/00e11f480847a4e88533f1e95b7c626a213d780b/src/bosh_aws_cpi/lib/cloud/aws/cloud_v2.rb#L41) ## Related - * [create_vm V1](../cpi-api-v1-method/create-vm.md) - * [delete_vm](delete-vm.md) +- [create_vm V1](../cpi-api-v1-method/create-vm.md) +- [delete_vm](delete-vm.md) diff --git a/content/cpi-api-v2-method/delete-disk.md b/content/cpi-api-v2-method/delete-disk.md index 049e05588..30f462fab 100644 --- a/content/cpi-api-v2-method/delete-disk.md +++ b/content/cpi-api-v2-method/delete-disk.md @@ -4,26 +4,21 @@ Deletes disk. Assume that disk was detached from all VMs. To avoid losing track of disks, make sure to raise an error if disk deletion is not absolutely certain. - ## Arguments - * `disk_cid` [String]: Cloud ID of the disk to delete; returned from `create_disk`. - +- `disk_cid` [String]: Cloud ID of the disk to delete; returned from `create_disk`. ## Result No return value - ## Examples - ### Implementations - * [cloudfoundry/bosh-warden-cpi-release](https://github.com/cloudfoundry/bosh-warden-cpi-release/blob/master/src/bosh-warden-cpi/action/delete_disk.go) - +- [cloudfoundry/bosh-warden-cpi-release](https://github.com/cloudfoundry/bosh-warden-cpi-release/blob/master/src/bosh-warden-cpi/action/delete_disk.go) ## Related - * [detach_disk](detach-disk.md) - * [create_disk](create-disk.md) +- [detach_disk](detach-disk.md) +- [create_disk](create-disk.md) diff --git a/content/cpi-api-v2-method/delete-network.md b/content/cpi-api-v2-method/delete-network.md index 854ca3287..3153201f6 100644 --- a/content/cpi-api-v2-method/delete-network.md +++ b/content/cpi-api-v2-method/delete-network.md @@ -4,14 +4,12 @@ Deletes a network that was created using `create_network`. ## Arguments -* `network_id` [String]: network_id of the network to delete. - +- `network_id` [String]: network_id of the network to delete. ## Result No return value. - ## Examples ### API request @@ -27,12 +25,10 @@ No return value. } ``` - ### Implementations - * [cloudfoundry/bosh-vsphere-cpi-release](https://github.com/cloudfoundry/bosh-vsphere-cpi-release/blob/dfe878579cbab768af07a12bb5543cd016cbb762/src/vsphere_cpi/lib/cloud/vsphere/cloud.rb#L690) - +- [cloudfoundry/bosh-vsphere-cpi-release](https://github.com/cloudfoundry/bosh-vsphere-cpi-release/blob/dfe878579cbab768af07a12bb5543cd016cbb762/src/vsphere_cpi/lib/cloud/vsphere/cloud.rb#L690) ## Related - * [create_network](create-network.md) +- [create_network](create-network.md) diff --git a/content/cpi-api-v2-method/delete-snapshot.md b/content/cpi-api-v2-method/delete-snapshot.md index 62594b259..458bb984b 100644 --- a/content/cpi-api-v2-method/delete-snapshot.md +++ b/content/cpi-api-v2-method/delete-snapshot.md @@ -2,17 +2,14 @@ Deletes the disk snapshot. - ## Arguments - * `snapshot_cid` [String]: Cloud ID of the disk snapshot. - +- `snapshot_cid` [String]: Cloud ID of the disk snapshot. ## Result No return value - ## Related - * [snapshot_disk](snapshot-disk.md) +- [snapshot_disk](snapshot-disk.md) diff --git a/content/cpi-api-v2-method/delete-stemcell.md b/content/cpi-api-v2-method/delete-stemcell.md index 0b7416066..534a11410 100644 --- a/content/cpi-api-v2-method/delete-stemcell.md +++ b/content/cpi-api-v2-method/delete-stemcell.md @@ -2,25 +2,20 @@ Deletes previously created stemcell. Assume that none of the VMs require presence of the stemcell. - ## Arguments - * `stemcell_cid` [String]: Cloud ID of the stemcell to delete; returned from `create_stemcell`. - +- `stemcell_cid` [String]: Cloud ID of the stemcell to delete; returned from `create_stemcell`. ## Result No return value - ## Examples - ### Implementations - * cloudfoundry/bosh-warden-cpi-release](https://github.com/cloudfoundry/bosh-warden-cpi-release/blob/master/src/bosh-warden-cpi/action/delete_stemcell.go) - +- cloudfoundry/bosh-warden-cpi-release]() ## Related - * [create_stemcell](create-stemcell.md) +- [create_stemcell](create-stemcell.md) diff --git a/content/cpi-api-v2-method/delete-vm.md b/content/cpi-api-v2-method/delete-vm.md index c61776066..d2993caa7 100644 --- a/content/cpi-api-v2-method/delete-vm.md +++ b/content/cpi-api-v2-method/delete-vm.md @@ -6,17 +6,14 @@ This method will be called while the VM still has persistent disks attached. It To avoid losing track of VMs, make sure to raise an error if VM deletion is not absolutely certain. - ## Arguments - **vm_cid** [String]: Cloud ID of the VM to delete; returned from `create_vm`. - ## Result No return value - ## Examples ### API request @@ -37,14 +34,13 @@ No return value ### Implementations - * [bosh-warden-cpi-release](https://github.com/cloudfoundry/bosh-warden-cpi-release/blob/master/src/bosh-warden-cpi/action/delete_vm.go) +- [bosh-warden-cpi-release](https://github.com/cloudfoundry/bosh-warden-cpi-release/blob/master/src/bosh-warden-cpi/action/delete_vm.go) #### Changes for V2 of the API contract The signature for `delete_vm` is the same as in V1 pf the API contract. For CPIs that previously used the registry to track mount points, V2 does not necessarily use the registry. The registry is used if the stemcell API version is not sufficient. The Agent receives an `unmount_disk` from the Director and deletes the persistent disk settings when not using the registry. See [CPI V2 Migration Guide](../cpi-api-v2-migration-guide.md) for more information. - ## Related - * [delete_vm V1](../cpi-api-v1-method/delete-vm.md) - * [create_vm](create-vm.md) +- [delete_vm V1](../cpi-api-v1-method/delete-vm.md) +- [create_vm](create-vm.md) diff --git a/content/cpi-api-v2-method/detach-disk.md b/content/cpi-api-v2-method/detach-disk.md index b35fdaa8b..c853163c9 100644 --- a/content/cpi-api-v2-method/detach-disk.md +++ b/content/cpi-api-v2-method/detach-disk.md @@ -6,23 +6,19 @@ If the persistent disk is attached to a VM that will be deleted, it is more like Agent settings must have been updated to remove information about the given disk. - ## Arguments - * `vm_cid` [String]: Cloud ID of the VM. - * `disk_cid` [String]: Cloud ID of the disk. - +- `vm_cid` [String]: Cloud ID of the VM. +- `disk_cid` [String]: Cloud ID of the disk. ## Result No return value - ## Examples ### API request - ```json { "method": "detach_disk", @@ -31,7 +27,7 @@ No return value "vol-044c8ae985721d217" ], "context": { - "director_uuid": "", + "director_uuid": "", "request_id": "", "vm": { "stemcell": { @@ -53,10 +49,9 @@ No return value } ``` - ### Implementations - * [cloudfoundry/bosh-warden-cpi-release](https://github.com/cloudfoundry/bosh-warden-cpi-release/blob/master/src/bosh-warden-cpi/action/detach_disk.go) +- [cloudfoundry/bosh-warden-cpi-release](https://github.com/cloudfoundry/bosh-warden-cpi-release/blob/master/src/bosh-warden-cpi/action/detach_disk.go) #### Changes for V2 of the API contract @@ -64,6 +59,6 @@ The signature for `detach_disk` is the same as in V1 of the API contract. For CP ## Related - * [detach_disk V1](../cpi-api-v1-method/detach-disk.md) - * [attach_disk](attach-disk.md) - * [delete_disk](delete-disk.md) +- [detach_disk V1](../cpi-api-v1-method/detach-disk.md) +- [attach_disk](attach-disk.md) +- [delete_disk](delete-disk.md) diff --git a/content/cpi-api-v2-method/get-disks.md b/content/cpi-api-v2-method/get-disks.md index cc9fcf73c..7144197d8 100644 --- a/content/cpi-api-v2-method/get-disks.md +++ b/content/cpi-api-v2-method/get-disks.md @@ -4,18 +4,15 @@ Returns list of disks _currently_ attached to the VM. This method is mostly used by the consistency check tool (cloudcheck) to determine if the VM has required disks attached. - ## Arguments -* `vm_cid` [String]: Cloud ID of the VM. - +- `vm_cid` [String]: Cloud ID of the VM. ## Result -* `disk_cids` [Array of strings]: Array of `disk_cid`s that are currently attached to the VM. - +- `disk_cids` [Array of strings]: Array of `disk_cid`s that are currently attached to the VM. ## Related - * [create_disk](create-disk.md) - * [attach_disk](attach-disk.md) +- [create_disk](create-disk.md) +- [attach_disk](attach-disk.md) diff --git a/content/cpi-api-v2-method/gopher.jpg b/content/cpi-api-v2-method/gopher.jpg deleted file mode 100644 index 3162e1324e75a18fffd1b4cd023357cb12144bbe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1754 zcmbV~do)yA9LM*WGh=3`nDGkT@+^6D^C&_oJv2&1%4?<24C=a~V#=Kn-SQ}*Bq^0f zBqcO6$}>05Ql5EUukk3+&VGI_=;Y6>280bF0l5k`-gKZMBcJ6p}e>ua@xDNVPHb&NM`G%?*|wtdIWUDmrPHU}LJ9d>kbcJVw;rFnV#oH%{v zY+w-m-1&=tUb-9}5gB#;M*Pi$TZy-m9V}~NIjgvooKm8Ok^QK=M?k%#f~Ik=)|d!QO!jkNVgHM4 z5$p$72iOiUV!;^nlwcS=DJ&`+9w!o>fESCfAW5;LL|VEi1Py`E476DSeU=jO#Kpaz zHb^IWEip(tSOP=nWr9h-63ox7!IT&=Erjha=u(wtGokKz@f}%l8LdS9m773Og4*h2 zhVQB9Se0`WmPN%#EXD1Fux=H{E5^cr_tmRZNTPF2ZZ$0p!<8t!0h=wgHz_8%M>a5?_CL)x=CqF-oSB%m z<3D4Jh4-XI@}j9V`JBAlX~TvqXs&nm{+i?;n(y@1{227=8jzn1aS;YGam;!Ikc6^y zXXE{lTgUu8$Ka7Wj*(eS@7FtcR0X_V8U-Cmzd&?9nsYz5w(OY~&+$#|S3mMQ%jxuc z&OQ0o6nn!IL#b41+#_enZ!8!F@>$<*m^_M1dpyA$UwgEcmDB5Aa`{~CvW9EsN`yJ3 zx7FocEsl=1tFxZ_nw@Mw08%%fIQCCLI(<*MOW>o7_IE*^Plj$Km|T(Rs;`ubGt7?3 z45Lv`rJI%lUNM#b;+YS(WC^=F{656q{nR5n*jumvWsZ+0XNQ(2(NCk&xc5jX<3nKx zX|O1-uVZhXnfca&yUxQ}iq~9gI6cQh?^hJd@+*1!73rHNGn4W5OI)Xd2F<{tXYPtMmOxPdR z)V>gf;iReP!scdyZ*tM4lDjF&)Z{M5*z_*D_(KO;Ey&jDCFV@I5vKYW?e^0nBNCqu zy~<8njw=&{a-+#qS~PZjdG%gua^xBf?oLMSsGJ#>og4c>t#Kxpp@?}8V zxQ`n6F;&3vWlrX`UbV^GE|WcxlpxUU)NdA?Wau+YERp*8NPhm4v$oQzt0oy+nJUj} zC*bktFuyZxs09Jihfj_;t<&TZ`wMe2*{<0t?#FCC6&obYbYHnYE?0+ukxK}e$a^l$ zuS9_6;L+5h4MjDUb8M#9YH8uFJ^d%>mNt#XCAi-j^?KOO(-sd22?iQh-4nHywhR`9 z^b+6$Eu&`o5#Khx^`E^l+j5H^B-~L|(z$=gjWrrBv|6bua6-UmL%S#h1ToXd2ng{) G+WrMXT+2oP diff --git a/content/cpi-api-v2-method/has-disk.md b/content/cpi-api-v2-method/has-disk.md index 80a08b15f..90d5ca27a 100644 --- a/content/cpi-api-v2-method/has-disk.md +++ b/content/cpi-api-v2-method/has-disk.md @@ -4,25 +4,20 @@ Checks for disk presence in the IaaS. This method is mostly used by the consistency check tool (cloudcheck) to determine if the disk still exists. - ## Arguments - * `disk_cid` [String]: Cloud ID of the disk to check; returned from `create_disk`. - +- `disk_cid` [String]: Cloud ID of the disk to check; returned from `create_disk`. ## Result - * `exists` [Boolean]: True if disk is present. - +- `exists` [Boolean]: True if disk is present. ## Examples - ### Implementations - * [cloudfoundry/bosh-vsphere-cpi-release](https://github.com/cloudfoundry/bosh-vsphere-cpi-release/blob/dfe878579cbab768af07a12bb5543cd016cbb762/src/vsphere_cpi/lib/cloud/vsphere/cloud.rb#L129) - +- [cloudfoundry/bosh-vsphere-cpi-release](https://github.com/cloudfoundry/bosh-vsphere-cpi-release/blob/dfe878579cbab768af07a12bb5543cd016cbb762/src/vsphere_cpi/lib/cloud/vsphere/cloud.rb#L129) ## Related - * [create_disk](create-disk.md) +- [create_disk](create-disk.md) diff --git a/content/cpi-api-v2-method/has-vm.md b/content/cpi-api-v2-method/has-vm.md index d778768e2..5ce31489a 100644 --- a/content/cpi-api-v2-method/has-vm.md +++ b/content/cpi-api-v2-method/has-vm.md @@ -4,25 +4,20 @@ Checks for VM presence in the IaaS. This method is mostly used by the consistency check tool (cloudcheck) to determine if the VM still exists. - ## Arguments - * `vm_cid` [String]: Cloud ID of the VM to check; returned from `create_vm`. - +- `vm_cid` [String]: Cloud ID of the VM to check; returned from `create_vm`. ## Returned - * `exists` [Boolean]: True if VM is present. - +- `exists` [Boolean]: True if VM is present. ## Examples - ### Implementations - * [cloudfoundry/bosh-warden-cpi-release](https://github.com/cloudfoundry/bosh-warden-cpi-release/blob/master/src/bosh-warden-cpi/action/has_vm.go) - +- [cloudfoundry/bosh-warden-cpi-release](https://github.com/cloudfoundry/bosh-warden-cpi-release/blob/master/src/bosh-warden-cpi/action/has_vm.go) ## Related - * [create_vm](create-vm.md) +- [create_vm](create-vm.md) diff --git a/content/cpi-api-v2-method/info.md b/content/cpi-api-v2-method/info.md index 7c3171f35..40b63e57d 100644 --- a/content/cpi-api-v2-method/info.md +++ b/content/cpi-api-v2-method/info.md @@ -2,16 +2,14 @@ Returns information about the CPI to help the Director to make decisions on which CPI to call for certain operations in a multi CPI scenario. - ## Arguments No arguments - ## Result - * `stemcell_formats` [Array of strings]: The list of stemcell formats this CPI supports. - * `api_version` [int]: maximum version of the API contract supported by the CPI. +- `stemcell_formats` [Array of strings]: The list of stemcell formats this CPI supports. +- `api_version` [int]: maximum version of the API contract supported by the CPI. ## Examples @@ -42,8 +40,9 @@ No arguments } } ``` + The `api_version` is the version of the API contract that the CPI supports. New CPIs adopting the V2 contract must return `2`. If there is no version supplied, the Director assumes version 1 of the contract must be used. ## Related - * [info V1](../cpi-api-v1-method/info.md) +- [info V1](../cpi-api-v1-method/info.md) diff --git a/content/cpi-api-v2-method/reboot-vm.md b/content/cpi-api-v2-method/reboot-vm.md index a80c45988..b90913d09 100644 --- a/content/cpi-api-v2-method/reboot-vm.md +++ b/content/cpi-api-v2-method/reboot-vm.md @@ -4,20 +4,16 @@ Reboots the VM. Assume that VM can be either be powered on or off at the time of Waiting for the VM to finish rebooting is not required because the Director waits until the Agent on the VM responds back. - ## Arguments - * `vm_cid` [String]: Cloud ID of the VM to reboot; returned from `create_vm`. - +- `vm_cid` [String]: Cloud ID of the VM to reboot; returned from `create_vm`. ## Result No return value - ## Examples - ### Implementations - * [cloudfoundry/bosh-vsphere-cpi-release](https://github.com/cloudfoundry/bosh-vsphere-cpi-release/blob/dfe878579cbab768af07a12bb5543cd016cbb762/src/vsphere_cpi/lib/cloud/vsphere/cloud.rb#L409) +- [cloudfoundry/bosh-vsphere-cpi-release](https://github.com/cloudfoundry/bosh-vsphere-cpi-release/blob/dfe878579cbab768af07a12bb5543cd016cbb762/src/vsphere_cpi/lib/cloud/vsphere/cloud.rb#L409) diff --git a/content/cpi-api-v2-method/resize-disk.md b/content/cpi-api-v2-method/resize-disk.md index dbca07a38..56b098dd6 100644 --- a/content/cpi-api-v2-method/resize-disk.md +++ b/content/cpi-api-v2-method/resize-disk.md @@ -6,27 +6,23 @@ Depending on the capabilities of the underlying infrastructure, this method may If `Bosh::Clouds::NotSupported` is raised, the Director falls back to creating a new disk and copying data. - ## Arguments - * `disk_cid` [String]: Cloud ID of the disk to resize; returned from `create_disk`. - * `new_size` [Integer]: New disk size in MiB. - +- `disk_cid` [String]: Cloud ID of the disk to resize; returned from `create_disk`. +- `new_size` [Integer]: New disk size in MiB. ## Result No return value - ## Examples ### Implementations - * [cloudfoundry/bosh-openstack-cpi-release](https://github.com/cloudfoundry/bosh-openstack-cpi-release/blob/88e1c6d402b3c4ce23ad39ebdf5ab5fc93790127/src/bosh_openstack_cpi/lib/cloud/openstack/cloud.rb#L701) - * [cloudfoundry/bosh-aws-cpi-release](https://github.com/cloudfoundry/bosh-aws-cpi-release/blob/7906dc66c61d4667ad74beeda3e5627f997ad740/src/bosh_aws_cpi/lib/cloud/aws/cloud_core.rb#L165) - * [cloudfoundry/bosh-google-cpi-release](https://github.com/cloudfoundry/bosh-google-cpi-release/blob/f2d17fe164daa690acb37d5758b689a8cbc32852/src/bosh-google-cpi/action/concrete_factory.go#L180) - +- [cloudfoundry/bosh-openstack-cpi-release](https://github.com/cloudfoundry/bosh-openstack-cpi-release/blob/88e1c6d402b3c4ce23ad39ebdf5ab5fc93790127/src/bosh_openstack_cpi/lib/cloud/openstack/cloud.rb#L701) +- [cloudfoundry/bosh-aws-cpi-release](https://github.com/cloudfoundry/bosh-aws-cpi-release/blob/7906dc66c61d4667ad74beeda3e5627f997ad740/src/bosh_aws_cpi/lib/cloud/aws/cloud_core.rb#L165) +- [cloudfoundry/bosh-google-cpi-release](https://github.com/cloudfoundry/bosh-google-cpi-release/blob/f2d17fe164daa690acb37d5758b689a8cbc32852/src/bosh-google-cpi/action/concrete_factory.go#L180) ## Related - * [create_disk](create-disk.md) +- [create_disk](create-disk.md) diff --git a/content/cpi-api-v2-method/set-disk-metadata.md b/content/cpi-api-v2-method/set-disk-metadata.md index d2785c47d..a39072012 100644 --- a/content/cpi-api-v2-method/set-disk-metadata.md +++ b/content/cpi-api-v2-method/set-disk-metadata.md @@ -22,21 +22,17 @@ cannot be overridden. !!! note This `set_disk_metadata` method is called by BOSH v262+. - ## Arguments - * `disk_cid` [String]: Cloud ID of the disk to modify; returned from `create_disk`. - * `metadata` [Hash]: Collection of key-value pairs. CPI should not rely on presence of specific keys. - +- `disk_cid` [String]: Cloud ID of the disk to modify; returned from `create_disk`. +- `metadata` [Hash]: Collection of key-value pairs. CPI should not rely on presence of specific keys. ## Result No return value - ## Examples - ### API Request ```json @@ -54,12 +50,10 @@ No return value ] ``` - ### Implementations - * [cloudfoundry/bosh-openstack-cpi-release](https://github.com/cloudfoundry/bosh-openstack-cpi-release/blob/0c8ee8951cab41d0ddc86591719d55d8a783ac98/src/bosh_openstack_cpi/lib/cloud/openstack/cloud.rb#L629) - +- [cloudfoundry/bosh-openstack-cpi-release](https://github.com/cloudfoundry/bosh-openstack-cpi-release/blob/0c8ee8951cab41d0ddc86591719d55d8a783ac98/src/bosh_openstack_cpi/lib/cloud/openstack/cloud.rb#L629) ## Related - * [create_disk](create-disk.md) +- [create_disk](create-disk.md) diff --git a/content/cpi-api-v2-method/set-vm-metadata.md b/content/cpi-api-v2-method/set-vm-metadata.md index 4cce8d657..a5c1b94cd 100644 --- a/content/cpi-api-v2-method/set-vm-metadata.md +++ b/content/cpi-api-v2-method/set-vm-metadata.md @@ -32,18 +32,15 @@ the VM: ## Arguments - * `vm_cid` [String]: Cloud ID of the VM to modify; returned from `create_vm`. - * `metadata` [Hash]: Collection of key-value pairs, including the top-level `tags` in the deployment manifest. CPI should not rely on presence of specific keys. - +- `vm_cid` [String]: Cloud ID of the VM to modify; returned from `create_vm`. +- `metadata` [Hash]: Collection of key-value pairs, including the top-level `tags` in the deployment manifest. CPI should not rely on presence of specific keys. ## Result No return value - ## Examples - ### API Request ```json @@ -61,12 +58,10 @@ No return value ] ``` - ### Implementations - * [cloudfoundry/bosh-vsphere-cpi-release](https://github.com/cloudfoundry/bosh-vsphere-cpi-release/blob/dfe878579cbab768af07a12bb5543cd016cbb762/src/vsphere_cpi/lib/cloud/vsphere/cloud.rb#L433) - +- [cloudfoundry/bosh-vsphere-cpi-release](https://github.com/cloudfoundry/bosh-vsphere-cpi-release/blob/dfe878579cbab768af07a12bb5543cd016cbb762/src/vsphere_cpi/lib/cloud/vsphere/cloud.rb#L433) ## Related - * [create_vm](create-vm.md) +- [create_vm](create-vm.md) diff --git a/content/cpi-api-v2-method/snapshot-disk.md b/content/cpi-api-v2-method/snapshot-disk.md index e285aaf0c..23998665c 100644 --- a/content/cpi-api-v2-method/snapshot-disk.md +++ b/content/cpi-api-v2-method/snapshot-disk.md @@ -2,18 +2,15 @@ Takes a snapshot of the disk. - ## Arguments - * `disk_cid` [String]: Cloud ID of the disk. - * `metadata` [Hash]: Collection of key-value pairs. CPI should not rely on presence of specific keys. - +- `disk_cid` [String]: Cloud ID of the disk. +- `metadata` [Hash]: Collection of key-value pairs. CPI should not rely on presence of specific keys. ## Result - * `snapshot_cid` [String]: Cloud ID of the disk snapshot. - +- `snapshot_cid` [String]: Cloud ID of the disk snapshot. ## Related - * [create_disk](create-disk.md) +- [create_disk](create-disk.md) diff --git a/content/cpi-api-v2-method/update-disk.md b/content/cpi-api-v2-method/update-disk.md index 4147641c0..f41589acb 100644 --- a/content/cpi-api-v2-method/update-disk.md +++ b/content/cpi-api-v2-method/update-disk.md @@ -27,13 +27,13 @@ If the CPI does not support `update_disk`, the Director falls back to the standa ## Arguments - * `disk_cid` [String]: Cloud ID of the disk to update; returned from `create_disk`. - * `new_size` [Integer]: New disk size in MiB. - * `cloud_properties` [Hash]: New cloud properties for the disk. The properties are specific to the IaaS and are opaque to the Director. The Director does not validate the properties and passes them to the CPI as-is. +- `disk_cid` [String]: Cloud ID of the disk to update; returned from `create_disk`. +- `new_size` [Integer]: New disk size in MiB. +- `cloud_properties` [Hash]: New cloud properties for the disk. The properties are specific to the IaaS and are opaque to the Director. The Director does not validate the properties and passes them to the CPI as-is. ## Result - * `new_disk_cid` [String or nil]: If the disk was replaced (e.g., via snapshot and recreate due to an incompatible type change), returns the new disk's Cloud ID. The Director persists this new CID and uses it for subsequent operations. If the disk was updated in-place, the CPI may return `nil` or the original disk CID - in both cases, the Director continues using the existing CID. +- `new_disk_cid` [String or nil]: If the disk was replaced (e.g., via snapshot and recreate due to an incompatible type change), returns the new disk's Cloud ID. The Director persists this new CID and uses it for subsequent operations. If the disk was updated in-place, the CPI may return `nil` or the original disk CID - in both cases, the Director continues using the existing CID. ## Examples @@ -61,10 +61,10 @@ null ### Implementations - * [cloudfoundry/bosh-azure-cpi-release](https://github.com/cloudfoundry/bosh-azure-cpi-release/blob/fc16e6f65b0533f83052812dc0d6c1edefc9ac28/src/bosh_azure_cpi/lib/cloud/azure/cloud.rb#L482) - * [cloudfoundry/bosh-google-cpi-release](https://github.com/cloudfoundry/bosh-google-cpi-release/blob/master/src/bosh-google-cpi/action/update_disk.go) +- [cloudfoundry/bosh-azure-cpi-release](https://github.com/cloudfoundry/bosh-azure-cpi-release/blob/fc16e6f65b0533f83052812dc0d6c1edefc9ac28/src/bosh_azure_cpi/lib/cloud/azure/cloud.rb#L482) +- [cloudfoundry/bosh-google-cpi-release](https://github.com/cloudfoundry/bosh-google-cpi-release/blob/master/src/bosh-google-cpi/action/update_disk.go) ## Related - * [resize_disk](resize-disk.md) - * [create_disk](create-disk.md) \ No newline at end of file +- [resize_disk](resize-disk.md) +- [create_disk](create-disk.md) diff --git a/content/cpi-api-v2-migration-guide.md b/content/cpi-api-v2-migration-guide.md index e3e62e195..59d37b007 100644 --- a/content/cpi-api-v2-migration-guide.md +++ b/content/cpi-api-v2-migration-guide.md @@ -1,62 +1,64 @@ +# Migrating to Version 2 + ## Migrating from V1 to V2 of the CPI API contract -##### CPI Changes in V2 of the API contract: +### CPI Changes in V2 of the API contract - - CPI [`info`](cpi-api-v2-method/info.md) method exposes supported `api_version`. - - CPI [`create_vm`](cpi-api-v2-method/create-vm.md) method returns an array of vm_id, network_info. +- CPI [`info`](cpi-api-v2-method/info.md) method exposes supported `api_version`. +- CPI [`create_vm`](cpi-api-v2-method/create-vm.md) method returns an array of vm_id, network_info. - CPI adds agent settings, i.e. agent id, networks, disks (previously sent to registry) into VM metadata / user-data - - CPI [`attach_disk`](cpi-api-v2-method/attach-disk.md) method returns disk hints. - - CPI [`detach_disk`](cpi-api-v2-method/detach-disk.md) method has no changes. - - CPI [`delete_vm`](cpi-api-v2-method/delete-vm.md) method has no changes. +- CPI [`attach_disk`](cpi-api-v2-method/attach-disk.md) method returns disk hints. +- CPI [`detach_disk`](cpi-api-v2-method/detach-disk.md) method has no changes. +- CPI [`delete_vm`](cpi-api-v2-method/delete-vm.md) method has no changes. -##### V2 flow depending on registry availability (all examples are from `bosh-aws-cpi`) +### V2 flow depending on registry availability (all examples are from `bosh-aws-cpi`) - - `create_vm`: +- `create_vm`: - Returns `network_info`, that the director will use to perform additional tasks (future scoped for director). - Depending on which stemcell `api_version` the CPI receives in `context` - - when the stemcell `api_version` is **>= 2** - - The CPI does not update the registry. - - Adds the agent settings into the [user_metadata](https://github.com/cloudfoundry/bosh-aws-cpi-release/blob/f27c51db1930d1d4c12cbbf074962380377e9e74/src/bosh_aws_cpi/lib/cloud/aws/cloud_v2.rb#L45-L54) when it send the [create instance](https://github.com/cloudfoundry/bosh-aws-cpi-release/blob/f27c51db1930d1d4c12cbbf074962380377e9e74/src/bosh_aws_cpi/lib/cloud/aws/cloud_core.rb#L94-L102) request to the IaaS. - - when the stemcell `api_version` is **< 2** - - The CPI should call `update_registry` with all the required [agent settings](https://github.com/cloudfoundry/bosh-aws-cpi-release/blob/f27c51db1930d1d4c12cbbf074962380377e9e74/src/bosh_aws_cpi/lib/cloud/aws/cloud_v2.rb#L58-L60). This is the same as in V1. + - when the stemcell `api_version` is **>= 2** + - The CPI does not update the registry. + - Adds the agent settings into the [user_metadata](https://github.com/cloudfoundry/bosh-aws-cpi-release/blob/f27c51db1930d1d4c12cbbf074962380377e9e74/src/bosh_aws_cpi/lib/cloud/aws/cloud_v2.rb#L45-L54) when it send the [create instance](https://github.com/cloudfoundry/bosh-aws-cpi-release/blob/f27c51db1930d1d4c12cbbf074962380377e9e74/src/bosh_aws_cpi/lib/cloud/aws/cloud_core.rb#L94-L102) request to the IaaS. + - when the stemcell `api_version` is **< 2** + - The CPI should call `update_registry` with all the required [agent settings](https://github.com/cloudfoundry/bosh-aws-cpi-release/blob/f27c51db1930d1d4c12cbbf074962380377e9e74/src/bosh_aws_cpi/lib/cloud/aws/cloud_v2.rb#L58-L60). This is the same as in V1. - - `delete_vm`: +- `delete_vm`: - No changes in the API contract. - Depending on which stemcell `api_version` the CPI receives in `context` - - When the stemcell `api_version` is **>= 2** + - When the stemcell `api_version` is **>= 2** - The new CPI does not attempt to delete `instance_id` from the registry, as the `instance_id` will not exist in registry to begin with. - - When the stemcell `api_version` is **< 2** + - When the stemcell `api_version` is **< 2** - The CPI calls [Delete entry](https://github.com/cloudfoundry/bosh-aws-cpi-release/blob/f27c51db1930d1d4c12cbbf074962380377e9e74/src/bosh_aws_cpi/lib/cloud/aws/cloud_v2.rb#L110) with `instance_id`, which is the same behaviour as in V1. - - `attach_disk`: +- `attach_disk`: - Returns `disk_hints`, which will be used by director to perform additional tasks - - The format of `disk_hints` did not change; it is the same as the values put into the registry in the context of the V1 contract. - - Examples: - ``` - Older CPIs update settings with disk settings as strings - e.g "/dev/sdc" - "3" - Newer CPIs returns settings as a hash: - e.g {"path" => "/dev/sdc"} - {"volume_id" => "3"} - {"lun" => "0", "host_device_id" => "{host-device-id}"} - ``` - - Depending on which stemcell `api_version` the CPI receives in `context` - - When the stemcell `api_version` is **>= 2** - - The CPI does not try to update the registry with `disk_hints`. - - When the stemcell `api_version` is **< 2** - - The CPI [updates the registry](https://github.com/cloudfoundry/bosh-aws-cpi-release/blob/1d7c31ec1ea0bb65a287adfc1898810a615218b8/src/bosh_aws_cpi/lib/cloud/aws/cloud_v2.rb#L76-L80) with `disk_hints`, which is the same behaviour as in V1. + - The format of `disk_hints` did not change; it is the same as the values put into the registry in the context of the V1 contract. + - Examples: + ```text + Older CPIs update settings with disk settings as strings + e.g "/dev/sdc" + "3" + Newer CPIs returns settings as a hash: + e.g {"path" => "/dev/sdc"} + {"volume_id" => "3"} + {"lun" => "0", "host_device_id" => "{host-device-id}"} + ``` + - Depending on which stemcell `api_version` the CPI receives in `context` + - When the stemcell `api_version` is **>= 2** + - The CPI does not try to update the registry with `disk_hints`. + - When the stemcell `api_version` is **< 2** + - The CPI [updates the registry](https://github.com/cloudfoundry/bosh-aws-cpi-release/blob/1d7c31ec1ea0bb65a287adfc1898810a615218b8/src/bosh_aws_cpi/lib/cloud/aws/cloud_v2.rb#L76-L80) with `disk_hints`, which is the same behaviour as in V1. - - `detach_disk`: +- `detach_disk`: - No changes in the API contract - Depending on which stemcell `api_version` the CPI receives in `context` - - When the stemcell `api_version` is **>= 2** - - The CPI does not try to delete the `disk_id` entry in the agent settings inside the registry. - - When the stemcell `api_version` is **< 2** - - The CPI [deletes](https://github.com/cloudfoundry/bosh-aws-cpi-release/blob/1d7c31ec1ea0bb65a287adfc1898810a615218b8/src/bosh_aws_cpi/lib/cloud/aws/cloud_v2.rb#L94-L98) `disk_id` from the agent settings inside the registry, which is the same behaviour as in V1. + - When the stemcell `api_version` is **>= 2** + - The CPI does not try to delete the `disk_id` entry in the agent settings inside the registry. + - When the stemcell `api_version` is **< 2** + - The CPI [deletes](https://github.com/cloudfoundry/bosh-aws-cpi-release/blob/1d7c31ec1ea0bb65a287adfc1898810a615218b8/src/bosh_aws_cpi/lib/cloud/aws/cloud_v2.rb#L94-L98) `disk_id` from the agent settings inside the registry, which is the same behaviour as in V1. -##### Agent changes in V2 of the API contract: +### Agent changes in V2 of the API contract - The BOSH agent will leverage the IaaS' metadata service to obtain its settings (for `settings.json`) before falling back to the registry (if the full settings were not fetched or if there is no `agent_id` in the current settings). @@ -64,8 +66,7 @@ - The `unmount_disk` action unmounts the disk according to what is stored in `persistent_disk_hints.json` and then removes the disk entry from the file. - The `update_persistent_disk` action stores disk hints locally in `persistent_disk_hints.json`. - -##### Stemcell changes in V2 of the API contract: +### Stemcell changes in V2 of the API contract `stemcell.MF` must contain an `api_version: 2` entry if the stemcell has a V2-compatible agent installed. This will enable the director, CPI and cli to run in registry-less mode. If the entry is missing, the agent will fallback to the V1 contract and use the registry. @@ -86,26 +87,29 @@ cloud_properties: us-west-1: ami-xxxxxx ``` -### Updating existing CPIs for registry-less operation +## Updating existing CPIs for registry-less operation + !!! note CPIs implementing the V2 contract must also fully support the V1 API contract. -##### Ruby ![](https://cdn.emojidex.com/emoji/mdpi/Ruby.png) +### Ruby - Update the CPI Ruby gem to [v2.5.0](https://github.com/cloudfoundry/bosh-cpi-ruby/releases/tag/v2.5.0) - ``` - gem install bosh_cpi -v 2.5.0 - ``` + ```shell + gem install bosh_cpi -v 2.5.0 + ``` - For reference code you can check the updated [cloud_v2.rb](https://github.com/cloudfoundry/bosh-aws-cpi-release/blob/f27c51db1930d1d4c12cbbf074962380377e9e74/src/bosh_aws_cpi/lib/cloud/aws/cloud_v2.rb) in bosh-aws-cpi. -##### GoLang ![](cpi-api-v2-method/gopher.jpg) +### GoLang - Update [the CPI GO library](https://github.com/cloudfoundry/bosh-cpi-go) to the latest version: - ``` - go get -u github.com/cloudfoundry/bosh-cpi-go - ``` + + ```shell + go get -u github.com/cloudfoundry/bosh-cpi-go + ``` + - For reference code, see these CPIs using the library: - [Warden CPI](https://github.com/cloudfoundry/bosh-warden-cpi-release) - [VirtualBox CPI](https://github.com/cloudfoundry/bosh-virtualbox-cpi-release) @@ -115,6 +119,7 @@ cloud_properties: --- ### Reference pipeline to test a CPI with all combinations of the Director, CLI and Stemcell + This pipeline will use all permutations of the V1 and V2 contracts for the director, CLI and stemcell: [Pipeline for CPI V2 testing](https://github.com/cloudfoundry/bosh-aws-cpi-release/blob/49447ba7ee208c31dddc1b7e3ec2a5f05c88ea99/ci/pipeline_cpi_v2.yml.erb) @@ -122,14 +127,15 @@ This pipeline will use all permutations of the V1 and V2 contracts for the direc - Which version of the cpi API is specified in `cpi.json`. - Director using the V1 or V2 API contract - - For testing, the director is specifying [`director.cpi_api_test_max_version`](https://github.com/cloudfoundry-incubator/bosh-cpi-certification/blob/82dcf1843a1c617e73b59e4640af2090e9e0c37f/aws/assets/ops/director_cpi_version.yml) in its properties to use only the specified version of CPI contract. + - For testing, the director is specifying [`director.cpi_api_test_max_version`](https://github.com/cloudfoundry-incubator/bosh-cpi-certification/blob/82dcf1843a1c617e73b59e4640af2090e9e0c37f/aws/assets/ops/director_cpi_version.yml) in its properties to use only the specified version of CPI contract. - On the CPI side you can specify `debug.cpi.api_version` for debugging. Examples: - - [spec](https://github.com/cloudfoundry/bosh-aws-cpi-release/blob/f27c51db1930d1d4c12cbbf074962380377e9e74/jobs/aws_cpi/spec#L14-L16) - - ```yaml - debug.cpi.api_version: - description: api_version supported by cpi (can be used as an override for fallback). - default: null - ``` - - [cpi.json](https://github.com/cloudfoundry/bosh-aws-cpi-release/blob/f27c51db1930d1d4c12cbbf074962380377e9e74/jobs/aws_cpi/templates/cpi.json.erb#L34-L38) - - [config.rb](https://github.com/cloudfoundry/bosh-aws-cpi-release/blob/1d7c31ec1ea0bb65a287adfc1898810a615218b8/src/bosh_aws_cpi/lib/cloud/aws/config.rb#L75-L109) to load debug version if specified. + - [spec](https://github.com/cloudfoundry/bosh-aws-cpi-release/blob/f27c51db1930d1d4c12cbbf074962380377e9e74/jobs/aws_cpi/spec#L14-L16) + + ```yaml + debug.cpi.api_version: + description: api_version supported by cpi (can be used as an override for fallback). + default: null + ``` + + - [cpi.json](https://github.com/cloudfoundry/bosh-aws-cpi-release/blob/f27c51db1930d1d4c12cbbf074962380377e9e74/jobs/aws_cpi/templates/cpi.json.erb#L34-L38) + - [config.rb](https://github.com/cloudfoundry/bosh-aws-cpi-release/blob/1d7c31ec1ea0bb65a287adfc1898810a615218b8/src/bosh_aws_cpi/lib/cloud/aws/config.rb#L75-L109) to load debug version if specified. diff --git a/content/cpi-api-v2.md b/content/cpi-api-v2.md index 2e213d302..fbf1314e6 100644 --- a/content/cpi-api-v2.md +++ b/content/cpi-api-v2.md @@ -10,18 +10,16 @@ Examples of API request and response: - [Building a CPI: RPC - Request](https://bosh.io/docs/build-cpi.html#request) - [Building a CPI: RPC - Response](https://bosh.io/docs/build-cpi.html#response) - If you're looking to get started on building a CPI, this [short guide](build-cpi.md) may be helpful. To learn more about the technical implementation, continue reading or refer to the [RPC Interface](cpi-api-rpc.md) for more details. - Libraries: - Ruby: `bosh-cpi-ruby` gem [v2.5.0](https://github.com/cloudfoundry/bosh-cpi-ruby/releases/tag/v2.5.0) - GoLang: `bosh-cpi-go` [library](https://github.com/cloudfoundry/bosh-cpi-go) -#### Migration from V1 of the CPI API contract +## Migration from V1 of the CPI API contract -Detailed instructions on how to migrate an existing CPI from V1 to V2 of the contract can be found [here](cpi-api-v2-migration-guide.md). +Detailed instructions on how to migrate an existing CPI from V1 to V2 of the contract can be found in the [migration guide](cpi-api-v2-migration-guide.md). --- @@ -36,8 +34,8 @@ resource_pools: - name: large_machines cloud_properties: {instance_type: r3.8xlarge} ``` -`instance_type` is specific to AWS in this example, and is meaningless in the context of other CPIs. +`instance_type` is specific to AWS in this example, and is meaningless in the context of other CPIs. ## Methods @@ -49,16 +47,16 @@ resource_pools: For registry-less operation to be possible, the director and CPI must implement v2 of the API contract and the stemcell must contain a version of the agent implementing v2 as well. In the table below, `user-metadata` refers to the agent settings sent with `create_vm`. -| Director | CPI | Stemcell | Should update registry | Add *full agent settings** to IaaS `user-metadata`? | -|----------|-----|-----------|----------------------|---| -| 1 | 1 | 1 | Update registry | No | -| 1 | 1 | 2 | Update registry | No | -| 1 | 2 | 2 | Update registry | No | -| **2** | **2** | **2** | **Do not write to registry** | **Yes** | -| 1 | 2 | 1 | Update registry | No | -| 2 | 2 | 1 | Update registry | No | -| 2 | 1 | 1 | Update registry | No | -| 2 | 1 | 2 | Update registry | No | +| Director | CPI | Stemcell | Should update registry | Add *full agent settings** to IaaS `user-metadata`? | +|----------|-------|----------|------------------------------|-----------------------------------------------------| +| 1 | 1 | 1 | Update registry | No | +| 1 | 1 | 2 | Update registry | No | +| 1 | 2 | 2 | Update registry | No | +| **2** | **2** | **2** | **Do not write to registry** | **Yes** | +| 1 | 2 | 1 | Update registry | No | +| 2 | 2 | 1 | Update registry | No | +| 2 | 1 | 1 | Update registry | No | +| 2 | 1 | 2 | Update registry | No | \* see below for information on the settings to write to `user-metadata`. @@ -66,7 +64,8 @@ For registry-less operation to be possible, the director and CPI must implement When using the registry, the agent needs a minimal set of settings on bootstrap, including the registry location. When operating in registry-less mode, a more complete set of information is given to the agent via `user-metadata` through `create_vm`. The contents of `user-metadata` are IaaS-specific. For example, AWS uses the `user_data` field during instance creation. -**Registry is used, CPI contract V1 and V2 (see above for conditions)** +#### Registry is used, CPI contract V1 and V2 (see above for conditions) + ```json { "dns": {}, @@ -78,9 +77,11 @@ When using the registry, the agent needs a minimal set of settings on bootstrap, } } ``` + The remainder of the required settings are then fetched from the registry. The CPI has already written disk settings, etc. -**Registry is bypassed, CPI contract V2 only** +#### Registry is bypassed, CPI contract V2 only + ```json { "agent_id": "...", @@ -96,6 +97,7 @@ The remainder of the required settings are then fetched from the registry. The C } } ``` + **Note** that the `persistent` disks will not be available when the CPI writes these settings. When the Director instructs the CPI to attach a disk, the `attach_disk` method is expected to return information on the attach point. The Director then informs the Agent about the disk, and the Agent updates its disk settings accordingly. ### API contract changes since V1 @@ -109,11 +111,11 @@ CPI contract version 2 differs from version 1 by the following: #### API method changes -* General - * [info](cpi-api-v2-method/info.md) -* VM Management - * [create_vm](cpi-api-v2-method/create-vm.md) - * [delete_vm](cpi-api-v2-method/delete-vm.md) -* Disk Management - * [attach_disk](cpi-api-v2-method/attach-disk.md) - * [detach_disk](cpi-api-v2-method/detach-disk.md) +- General + - [info](cpi-api-v2-method/info.md) +- VM Management + - [create_vm](cpi-api-v2-method/create-vm.md) + - [delete_vm](cpi-api-v2-method/delete-vm.md) +- Disk Management + - [attach_disk](cpi-api-v2-method/attach-disk.md) + - [detach_disk](cpi-api-v2-method/detach-disk.md) diff --git a/content/cpi-api-v3-method/create-stemcell.md b/content/cpi-api-v3-method/create-stemcell.md index 987270cf5..bb378fb78 100644 --- a/content/cpi-api-v3-method/create-stemcell.md +++ b/content/cpi-api-v3-method/create-stemcell.md @@ -4,44 +4,40 @@ Creates a reusable VM image in the IaaS from the [stemcell](../stemcell.md) imag See [Stemcell Building](../build-stemcell.md) for more details. - ## Arguments - * `image_path` [String]: Path to the stemcell image extracted from the stemcell tarball on a local filesystem. - * `cloud_properties` [Hash]: Cloud properties hash extracted from the stemcell tarball. - * `env` [Hash]: A random hash with a "tags" key which itself is a hash of key/value pairs. This is used from version 282.0.0 of the director. - +- `image_path` [String]: Path to the stemcell image extracted from the stemcell tarball on a local filesystem. +- `cloud_properties` [Hash]: Cloud properties hash extracted from the stemcell tarball. +- `env` [Hash]: A random hash with a "tags" key which itself is a hash of key/value pairs. This is used from version 282.0.0 of the director. ## Result - * `stemcell_cid` [String]: Cloud ID of the created stemcell (e.g. stemcells in AWS CPI are made into AMIs so cid .would be `ami-83fdflf`) - +- `stemcell_cid` [String]: Cloud ID of the created stemcell (e.g. stemcells in AWS CPI are made into AMIs so cid .would be `ami-83fdflf`) ## Example - ### API Request ```json [ - "/tmp/extracted-stemcell-348754vdsn87fr/image", - { - "name": "bosh-openstack-esxi-ubuntu-xenial-go_agent", - "version": "621.74", - "infrastructure": "openstack", - "hypervisor": "esxi", - "disk": 3072, - "disk_format": "ovf", - "container_format": "bare", - "os_type": "linux", - "os_distro": "ubuntu", - "architecture": "x86_64", - "auto_disk_config": true - }, - {"tags": {"any": "tag value"} } + "/tmp/extracted-stemcell-348754vdsn87fr/image", + { + "name": "bosh-openstack-esxi-ubuntu-xenial-go_agent", + "version": "621.74", + "infrastructure": "openstack", + "hypervisor": "esxi", + "disk": 3072, + "disk_format": "ovf", + "container_format": "bare", + "os_type": "linux", + "os_distro": "ubuntu", + "architecture": "x86_64", + "auto_disk_config": true + }, + {"tags": {"any": "tag value"} } ] ``` ### Implementations - * [cloudfoundry/bosh-aws-cpi-release](https://github.com/cloudfoundry/bosh-warden-cpi-release/blob/master/src/bosh-warden-cpi/action/create_stemcell.go) +- [cloudfoundry/bosh-aws-cpi-release](https://github.com/cloudfoundry/bosh-warden-cpi-release/blob/master/src/bosh-warden-cpi/action/create_stemcell.go) diff --git a/content/cpi-api-v3.md b/content/cpi-api-v3.md index 1360b38d7..4e78470ca 100644 --- a/content/cpi-api-v3.md +++ b/content/cpi-api-v3.md @@ -10,10 +10,8 @@ Examples of API request and response: - [Building a CPI: RPC - Request](https://bosh.io/docs/build-cpi.html#request) - [Building a CPI: RPC - Response](https://bosh.io/docs/build-cpi.html#response) - If you're looking to get started on building a CPI, this [short guide](build-cpi.md) may be helpful. To learn more about the technical implementation, continue reading or refer to the [RPC Interface](cpi-api-rpc.md) for more details. - Libraries: - Ruby: `bosh-cpi-ruby` gem [v2.5.0](https://github.com/cloudfoundry/bosh-cpi-ruby/releases/tag/v2.5.0) @@ -30,5 +28,5 @@ Libraries: This list of methods to expect an additional parameter: -* Stemcell Management - * [create_stemcell](cpi-api-v3-method/create-stemcell.md) \ No newline at end of file +- Stemcell Management + - [create_stemcell](cpi-api-v3-method/create-stemcell.md) diff --git a/content/cpi-config.md b/content/cpi-config.md index 6517541eb..9027699bd 100644 --- a/content/cpi-config.md +++ b/content/cpi-config.md @@ -1,3 +1,5 @@ +# Using the CPI Config + !!! note This feature is available with bosh-release v261+. @@ -6,6 +8,7 @@ In most cases having single Director use a single CPI and hence a single IaaS se The CPI config is a YAML file that defines multiple CPIs and properties necessary for each CPI to communicate with an appropriate IaaS section. Once CPIs are specified, operator can associate particular AZ in their cloud config to a particular CPI. --- + ## Updating and retrieving CPI config {: #update } To update CPI config on the Director use [`bosh update-cpi-config`](cli-v2.md#update-cpi-config) CLI command. @@ -18,7 +21,7 @@ bosh update-cpi-config cpis.yml bosh cpi-config ``` -```text +```yaml Using environment '192.168.56.6' as client 'admin' cpis: @@ -35,13 +38,14 @@ cpis: Once CPI config is updated AZs in the cloud config can reference specific CPI to be used during a deploy. Unlike runtime and cloud configs, CPI config is not tracked directly by the deployments and can be updated separately (useful for updating CPI credentials without forcing redeploy of all the deployments). --- + ## CPIs Block {: #cpis } **cpis** [Array, required]: Specifies the CPIs. -* **name** [String, required]: Unique name for a CPI. Example: `openstack-1a`. -* **type** [String, required]: CPI type. Director will add `_cpi` suffix to the end of the type when calling the CPI binary. Example: `openstack`, `google`. -* **properties** [Hash, required]: Set of properties to provide to the CPI for each call so that CPI can authenticate and provision resources in an IaaS. +- **name** [String, required]: Unique name for a CPI. Example: `openstack-1a`. +- **type** [String, required]: CPI type. Director will add `_cpi` suffix to the end of the type when calling the CPI binary. Example: `openstack`, `google`. +- **properties** [Hash, required]: Set of properties to provide to the CPI for each call so that CPI can authenticate and provision resources in an IaaS. !!! note Properties will vary depending on the CPI you're trying to use. See the "Global Configuration" section of your CPI for more details (e.g. [AWS](aws-cpi.md#global), [vSphere](vsphere-cpi.md#global)). @@ -88,6 +92,7 @@ cpis: For vSphere, if your datacenter and cluster names have spaces in them, there is no need to put quotes around them when updating your cpi-config. --- + ## Example {: #example } Example of a CPI config referencing two separate OpenStack installations: @@ -172,7 +177,9 @@ azs: ``` --- + ## CPI Specific Stemcells {: #stemcells } + Stemcells need to be assigned to a specific CPI and it occurs on upload. If you've already uploaded an appropriate stemcell you'll need to re-upload with `--fix` ```shell diff --git a/content/create-release.md b/content/create-release.md index 5de0e51c3..84945c3c9 100644 --- a/content/create-release.md +++ b/content/create-release.md @@ -1,13 +1,15 @@ +# Creating a Release + A release contains one or more pieces of software that work together. For example, you could create a release of a service with three pieces: two MySQL nodes and a dashboard app. There are four fundamental elements in a release: -* **Jobs** describe pieces of the service or application you are releasing -* **Packages** provide source code and dependencies to jobs -* **Source** provides non-binary files to packages -* **Blobs** provide binary files (other than those checked into a source code repository) to packages +- **Jobs** describe pieces of the service or application you are releasing +- **Packages** provide source code and dependencies to jobs +- **Source** provides non-binary files to packages +- **Blobs** provide binary files (other than those checked into a source code repository) to packages The following instructions use an example release that includes two jobs: a web UI and a background worker. @@ -15,6 +17,7 @@ The two jobs split up the functionality provided by single Ruby app, `ardo_app` (you can use simple [gist](https://gist.github.com/antonsoroko/974924e0692aa2171229dafa5f2561b2) as app). --- + ## Preparation {: #prep } This section needs to be completed once. @@ -68,6 +71,7 @@ a Git submodule or a Mercurial repo. In our example, we create a folder named `ardo_app` and put our source code there. View the release with `tree`: + ```shell $ tree . . @@ -93,6 +97,7 @@ For releases with just a few jobs, going one step at a time is probably easiest. If you have a larger number of jobs, going one job at a time may be more efficient. --- + ## Step 1: Create Job Skeletons {: #job-skel } Navigate into the release directory. @@ -144,8 +149,8 @@ You provide that by writing a control script and updating the `monit` file. The control script: -* Includes a start command and a stop command. -* Is an ERB template stored in the `templates` directory for the relevant job. +- Includes a start command and a stop command. +- Is an ERB template stored in the `templates` directory for the relevant job. For each job, create a control script that configures the job to store logs in `/var/vcap/sys/log/JOB_NAME`. Save this script as `ctl.erb` in the `templates` directory for its job. @@ -192,7 +197,7 @@ esac If your release needs templates other than the control script, create them now. For example if the job can be used to deploy clusters of nodes, especially in -the case of stateful clusters (e.g. a database or distributed data store), you +the case of stateful clusters (e.g. a database or distributed data store), you will want to write a [drain script](drain.md) for your job to ensure that the service is not affected by the rolling provisioning/update operations performed by BOSH. @@ -201,9 +206,9 @@ by BOSH. The `monit` file: -* Specifies the process ID (pid) file for the job -* References each command provided by the templates for the job -* Specifies that the job belongs to the `vcap` group +- Specifies the process ID (pid) file for the job +- References each command provided by the templates for the job +- Specifies that the job belongs to the `vcap` group On a deployed release, a BOSH Agent runs on each job VM. BOSH communicates with the Agent, which in turn executes commands in the @@ -213,7 +218,7 @@ The Agent does this using open source process monitoring software called The `monit` file for the `web_ui` job looks like this: -``` +```shell check process web_ui with pidfile /var/vcap/sys/run/web_ui/pid start program "/var/vcap/jobs/web_ui/bin/ctl start" @@ -225,7 +230,7 @@ Update the `monit` file for each of your jobs. Use `/var/vcap` paths as shown in the example. !!! note - BOSH requires a monit file for each job in a release. When developing a release, you can use an empty monit file to meet this requirement without having to first create a control script. + BOSH requires a `monit` file for each job in a release. When developing a release, you can use an empty `monit` file to meet this requirement without having to first create a control script. ### Update job specs {: #job-specs } @@ -237,8 +242,8 @@ resides in the job `spec` file. In the job `spec` file, the `templates` block contains key/value pairs where: -* Each key is template name -* Each value is the path to the corresponding file on a job VM +- Each key is template name +- Each value is the path to the corresponding file on a job VM The file paths that you provide for templates are relative to the `/var/vcap/jobs/` directory on the VM. @@ -277,21 +282,22 @@ If you used the `--git` option with `bosh init-release` (as recommended), the correct `.gitignore` file has been automatically created for you. --- + ## Step 2: Make Dependency Graphs {: #graph } There are two kinds of dependencies in a BOSH release: -* The **runtime dependency**, where a job depends on a package at runtime. +- The **runtime dependency**, where a job depends on a package at runtime. For example, the `web_ui` job depends on Ruby. -* The **compile-time dependency**, where a package depends on another package at +- The **compile-time dependency**, where a package depends on another package at compile time. For example, Ruby depends on the YAML library. Three rules govern these dependencies: -* Jobs never depend on other jobs. -* Jobs can depend on packages. -* Packages can depend on other packages. +- Jobs never depend on other jobs. +- Jobs can depend on packages. +- Packages can depend on other packages. ### Building the Dependency Graph {: #build-graph } @@ -308,13 +314,13 @@ Add these runtime dependencies to your dependency graph. In our example, this line in both of our `ctl.erb` scripts cites `ardo_app`: -``` +```shell cd /var/vcap/packages/ardo_app ``` This line cites Ruby: -``` +```shell exec /var/vcap/packages/ruby_1.9.3/bin/bundle exec ``` @@ -395,7 +401,7 @@ to create is a matter of preference. You could even opt to put all the dependencies together in a single package, though that is not recommended. !!! note - Use of the pre_packaging file is not recommended, and is not discussed in this tutorial. + Use of the `pre_packaging` file is not recommended, and is not discussed in this tutorial. Without using `pre_packaging` for our `ardo_app` we need to pack gems manually for further usage: @@ -408,9 +414,9 @@ bundle package Within each package directory, there is a `spec` file which states: -* The package name -* The package's dependencies -* The location where BOSH can find the binaries and other files that the package needs at compile time +- The package name +- The package's dependencies +- The location where BOSH can find the binaries and other files that the package needs at compile time Use your dependency graph to determine which dependencies belong in each spec. Developer preferences and style play a role here. @@ -423,12 +429,12 @@ never depend on the presence of libraries or other software on stemcells. To describe binary locations in the `files` block of the spec: -* Find the official site for the binary in question. +- Find the official site for the binary in question. For example, Ruby might be at `http://cache.ruby-lang.org/pub/ruby/1.9/ruby-1.9.3-p484.tar.gz`. -* Download the binary from the official location and make sure the file hash matches. +- Download the binary from the official location and make sure the file hash matches. -* Record the binary name including version number, with a slash and the binary +- Record the binary name including version number, with a slash and the binary filename concatenated to it. It's a good idea to cite the official URL in a comment, in the same line. @@ -496,30 +502,30 @@ The instructions may involve some combination of copying, compilation, and related procedures. For example: -* For a Ruby app like `ardo_app`, BOSH must copy source files and install Ruby +- For a Ruby app like `ardo_app`, BOSH must copy source files and install Ruby gems. -* For Ruby itself, BOSH must compile source code into a binary. +- For Ruby itself, BOSH must compile source code into a binary. -* For a Python app, BOSH must copy source files and install Python eggs. +- For a Python app, BOSH must copy source files and install Python eggs. BOSH relies on you to write packaging scripts that perform the correct operation. Adhere to these principles when writing packaging scripts: -* Use your dependency graph to determine which dependencies belong in each +- Use your dependency graph to determine which dependencies belong in each packaging script. -* Begin each script with a `set -e -x` line. +- Begin each script with a `set -e -x` line. This aids debugging at compile time by causing the script to exit immediately if a command exits with a non-zero exit code. -* Ensure that any copying, installing or compiling delivers resulting code to +- Ensure that any copying, installing or compiling delivers resulting code to the install target directory (represented as the `BOSH_INSTALL_TARGET` environment variable). For `make` commands, use `configure` or its equivalent to accomplish this. -* Be aware that BOSH ensures that dependencies cited in the `dependencies` +- Be aware that BOSH ensures that dependencies cited in the `dependencies` block of package `spec` files are available to the deployed binary. For example, in the `spec` file for the Ruby package, we cite libyaml as a dependency. @@ -543,7 +549,7 @@ Refer to the examples below for guidance. #### Example libyaml packaging script {: #pkg-script-libyaml } -``` +```shell set -e -x tar xzf libyaml_0.1.4/yaml-0.1.4.tar.gz @@ -557,7 +563,7 @@ popd #### Example Ruby packaging script {: #pkg-script-ruby } -``` +```shell set -e -x tar xzf ruby_1.9.3/ruby-1.9.3-p484.tar.gz @@ -581,7 +587,7 @@ ${BOSH_INSTALL_TARGET}/bin/gem install ruby_1.9.3/bundler-1.2.1.gem --no-ri --no #### Example ardo_app packaging script {: #pkg-script-ardo } -``` +```shell set -e -x cp -a ardo_app/* ${BOSH_INSTALL_TARGET} @@ -611,6 +617,7 @@ packages: ``` --- + ## Step 4: Add Blobs {: #blobs } When creating a release, you will likely use a source code repository. @@ -620,19 +627,19 @@ unsuited to dealing with large binaries (as is true of Git, for example). BOSH lets you avoid checking blobs into a repository by doing the following: -* For dev releases, use local copies of blobs. +- For dev releases, use local copies of blobs. -* For a final release, upload blobs to a blobstore, and direct BOSH to obtain the blobs from there. +- For a final release, upload blobs to a blobstore, and direct BOSH to obtain the blobs from there. ### Configure a blobstore {: #config-blobstore } In the `config` directory, you record the information BOSH needs about the blobstore: -* The `final.yml` file names the blobstore and declares its type, which is either `local` +- The `final.yml` file names the blobstore and declares its type, which is either `local` or one of several other types that specify blobstore providers. -* The `private.yml` file specifies the blobstore path, along with a secret. +- The `private.yml` file specifies the blobstore path, along with a secret. `private.yml` contains keys for accessing the blobstore and should not be checked into a repository. @@ -683,8 +690,8 @@ blobstore: If you have a `private.yml` file: -* **Required**: Include the `blobstore_path` in the `private.yml` file. -* **Optional**: Include the `blobstore_path` in the `final.yml` file. Doing so allows you to `gitignore` the `private.yml` file but still allow the release to be downloaded and used on other systems. +- **Required**: Include the `blobstore_path` in the `private.yml` file. +- **Optional**: Include the `blobstore_path` in the `final.yml` file. Doing so allows you to `gitignore` the `private.yml` file but still allow the release to be downloaded and used on other systems. !!! note The `blobstore_secret` is required for the `local` type blobstore. This is true even though the `blobstore_secret` line is deprecated and its content does not matter. There is never a `blobstore_secret` line for blobstores of types other than `local`. @@ -725,8 +732,8 @@ recognizes as BOSH blobs. The usage shown above works like this: -* For the first argument, you provide the path to the blob on your local system -* For the second argument, you provide a destination within the `blobs` directory +- For the first argument, you provide the path to the blob on your local system +- For the second argument, you provide a destination within the `blobs` directory in your release Using the package name as the prefix for the second argument of the `bosh add-blob` command @@ -797,6 +804,7 @@ When creating dev releases, do not run `bosh upload-blobs`. (You only run it when you do a final release.) --- + ## Step 5: Create Job Properties {: #create-props } If your service needs to be configurable at deployment time, @@ -814,7 +822,9 @@ relevant templates. For example, a start command can take a property as an argument, using the property lookup helper: - <%= p('') %> +```erb +<%= p('') %> +``` 1. Specify the property in the [deployment manifest](manifest-v2.md#instance-groups). @@ -833,6 +843,7 @@ properties: ``` --- + ## Step 6: Create a Dev Release {: #dev-release } All the elements needed to create a dev release should now be in place. @@ -898,14 +909,15 @@ logging output as you test your release. If your release fails tests, follow this pattern. -* Fix the code. -* Do a new dev release. -* Run `bosh deploy` to see whether the new release deploys successfully. +- Fix the code. +- Do a new dev release. +- Run `bosh deploy` to see whether the new release deploys successfully. Using `bosh deploy --recreate` can provide a clearer picture because with that option, BOSH deploys all the VMs from scratch. --- + ## Create a Final Release {: #final-release } Only proceed to this step if your latest dev release passes all tests. diff --git a/content/credhub-encryption-password-rotation.md b/content/credhub-encryption-password-rotation.md index 7264dc6fd..e95cc0e9e 100644 --- a/content/credhub-encryption-password-rotation.md +++ b/content/credhub-encryption-password-rotation.md @@ -1,14 +1,14 @@ # Rotating CredHub Encryption Password -### Preconditions +## Preconditions -* The director is in a healthy state. +- The director is in a healthy state. -### Assumptions +## Assumptions -* CredHub is co-located on the BOSH director VM +- CredHub is co-located on the BOSH director VM -### Step 1: Update CredHub to encrypt with new password {: #step-1} +## Step 1: Update CredHub to encrypt with new password {: #step-1} ```shell OLD_PWD=$(bosh interpolate --path=/credhub_encryption_password creds.yml) @@ -55,12 +55,12 @@ Ops file `add-old-credhub-encryption-password.yml`: provider_name: internal ``` -* create new password -* deactivate old password -* let CredHub decrypt all secrets with old password and encrypt all secrets with +- create new password +- deactivate old password +- let CredHub decrypt all secrets with old password and encrypt all secrets with new password -### Step 2: Update CredHub to remove old password {: #step-2} +## Step 2: Update CredHub to remove old password {: #step-2} ```shell cp creds.yml creds.yml.bak diff --git a/content/creds-tmpfs.md b/content/creds-tmpfs.md index d75b24d22..a17b904a7 100644 --- a/content/creds-tmpfs.md +++ b/content/creds-tmpfs.md @@ -1,3 +1,5 @@ +# Credentials on tmpfs + !!! tip "Beta Feature" This `tmpfs` config method was first introduced in bosh [v268.5.0](https://github.com/cloudfoundry/bosh/releases/tag/v268.5.0) and stemcell [Xenial 250](https://bosh.io/stemcells/#ubuntu-xenial). @@ -29,7 +31,7 @@ the message bus directly to the agent. For a complete solution, the the `tmpfs` agent feature. This feature can be enabled by setting the following property in your bosh director manifest: -``` +```yaml - name: bosh ... properties: @@ -43,15 +45,14 @@ If you are using a recent [bosh-deployment](https://github.com/cloudfoundry/bosh-deployment) then this is probably already enabled by default. - ## Caveats -* Due to the job configuration and credentials only being stored in-memory, it +- Due to the job configuration and credentials only being stored in-memory, it is not possible to successfully reboot VMs which have this feature enabled. VMs must instead be recreated in order to repopulate them with configuration and bring them back to a healthy state. -* The `tmpfs` is 100MB in size by default. We found that this leaves ample room +- The `tmpfs` is 100MB in size by default. We found that this leaves ample room for most job templates as they are normally small. However, there is a configuration option to increase the size of the allocated `tmpfs` disk if necessary. @@ -106,6 +107,7 @@ instance_groups: settings: tmpfs: true ``` + #### Run Dir Configuration The `run` dir is always kept on an in-memory `tmpfs` with a default size of 16MB. Override the default size of the `tmpfs` mount by setting the following `env` properties in your deployment manifest. diff --git a/content/deploy-config.md b/content/deploy-config.md index 2ddec3d11..654ed1d51 100644 --- a/content/deploy-config.md +++ b/content/deploy-config.md @@ -1,9 +1,12 @@ +# Using Deploy Config + !!! note This feature is available with CLI v7.5.0+. The Director has a way to specify global flags for all deploy commands. The deploy config is a YAML file that defines global flags that apply to all deployments. --- + ## Updating and retrieving deploy config {: #update } To update deploy config on the Director use [`bosh update-config --type deploy --name `](cli-v2.md#update-config) CLI command. @@ -32,6 +35,7 @@ include: The Director will apply the specified flags for the specified deployments during the next `bosh deploy` for that deployment. --- + ## Example {: #example1 } ```yaml @@ -42,10 +46,9 @@ include: - foo ``` -You can include and exclude deployments by using EITHER the `include` or `exclude` property. The [example](#example1) above will only apply the flags if the deployment "foo" gets deployed. +You can include and exclude deployments by using EITHER the `include` or `exclude` property. The [example](#example1) above will only apply the flags if the deployment "foo" gets deployed. In contrast, the [example](#example2) config below will ensure that only for the "foo" deployment the flags will NOT be applied. Only specifying the `flags` property will apply the flags for all deployments. - ```yaml flags: - recreate diff --git a/content/deploying-step-by-step.md b/content/deploying-step-by-step.md index e19283e05..ed62768ca 100644 --- a/content/deploying-step-by-step.md +++ b/content/deploying-step-by-step.md @@ -1,3 +1,5 @@ +# Deploying Step by Step + The Director will do the following steps when `bosh deploy` (or its related commands such as start, stop and recreate) command runs: 1. Check if there is a deployment with the name specified by the deployment manifest @@ -91,6 +93,7 @@ The Director will do the following steps when `bosh deploy` (or its related comm - issue get_state Agent call until job state is running or times out --- + ## Update and propagate DNS records {: #dns } 1. Create a new DNS records dataset and saves it to the blobstore diff --git a/content/deploying.md b/content/deploying.md index 47dd446ae..e9a752829 100644 --- a/content/deploying.md +++ b/content/deploying.md @@ -1,3 +1,5 @@ +# Deploying + Once the deployment manifest is complete and the referenced stemcells and releases are uploaded to the Director, we are ready to proceed with the deployment. The CLI has a single command to create and update a deployment: [`bosh deploy` command](cli-v2.md#deploy). From the Director perspective the same steps are taken to create or update a deployment. To create a Zookeeper deployment from `zookeeper.yml` deployment manifest run the deploy command: @@ -8,7 +10,7 @@ bosh -e vbox -d zookeeper deploy zookeeper.yml Should result in: -```text +```shell Using environment '192.168.56.6' as '?' Task 1133 @@ -39,7 +41,7 @@ bosh -e vbox -d zookeeper instances Should result in: -```text +```shell Using environment '192.168.56.6' as '?' Deployment 'zookeeper' diff --git a/content/deployment-basics.md b/content/deployment-basics.md index 9e0318c44..ea7589477 100644 --- a/content/deployment-basics.md +++ b/content/deployment-basics.md @@ -1,3 +1,5 @@ +# Building a Manifest + (See [What is a Deployment?](deployment.md) for an introduction to deployments.) A deployment is a collection of VMs, persistent disks and other resources. To create a deployment in the Director, it has to be described with a [deployment manifest](terminology.md#manifest). Most deployment manifests look something like this: @@ -57,7 +59,7 @@ Here is how deployment manifest describes a reasonably complex Zookeeper cluster 1. Zookepeer source code, configuration file, startup scripts - include `zookeeper` release version `0.0.5` to the `releases` section -1. Operating system image onto which install software +1. Operating system image onto which install software - include latest version of `ubuntu-xenial` stemcell 1. Create 5 Zookeeper VMs spread - add `zookeeper` [instance group](terminology.md#instance-group) with `instances: 5` diff --git a/content/deployment-convergence.md b/content/deployment-convergence.md index 856c95cb9..c8d382835 100644 --- a/content/deployment-convergence.md +++ b/content/deployment-convergence.md @@ -1,3 +1,5 @@ +# Deployment Convergence + During a deployment, BOSH tries to converge to the _intended_ state, _i.e._ the state described in the deployment manifest, from the current state. diff --git a/content/deployment-manifest.md b/content/deployment-manifest.md index 8615c1f24..e8e7df509 100644 --- a/content/deployment-manifest.md +++ b/content/deployment-manifest.md @@ -1,3 +1,5 @@ +# Deployment Manifest v1 + !!! note Once you opt into using cloud config all deployments must be converted to use manifest v2 format that disallows IaaS specific configuration. See [manifest v2 schema](manifest-v2.md) for allowed configurations. v257+ supports deploying both v1 and v2 manifests to the same director. @@ -5,19 +7,20 @@ The deployment manifest is a YAML file that defines the components and propertie Contents of a deployment manifest: -* [Deployment Identification](#deployment): A name for the deployment and the UUID of the Director managing the deployment -* [Releases Block](#releases): Name and version of each release in a deployment -* [Networks Block](#networks): Network configuration information -* [Resource Pools Block](#resource-pools): Properties of VMs that BOSH creates and manages -* [Disk Pools Block](#disk-pools): Properties of disk pools that BOSH creates and manages -* [Compilation Block](#compilation): Properties of compilation VMs -* [Update Block](#update): Defines how BOSH updates job instances during deployment -* [Jobs Block](#jobs): Configuration and resource information for jobs -* [Properties Block](#properties): Describes global properties and generalized configuration information +- [Deployment Identification](#deployment): A name for the deployment and the UUID of the Director managing the deployment +- [Releases Block](#releases): Name and version of each release in a deployment +- [Networks Block](#networks): Network configuration information +- [Resource Pools Block](#resource-pools): Properties of VMs that BOSH creates and manages +- [Disk Pools Block](#disk-pools): Properties of disk pools that BOSH creates and manages +- [Compilation Block](#compilation): Properties of compilation VMs +- [Update Block](#update): Defines how BOSH updates job instances during deployment +- [Jobs Block](#jobs): Configuration and resource information for jobs +- [Properties Block](#properties): Describes global properties and generalized configuration information The examples below originate from a [sample deployment manifest](sample-manifest.md). --- + ## Deployment Identification {: #deployment } **name** [String, required]: The name of the deployment. A single Director can manage multiple deployments and distinguishes them by name. @@ -33,12 +36,13 @@ director_uuid: cf8dc1fc-9c42-4ffc-96f1-fbad983a6ce6 ``` --- + ## Releases Block {: #releases } **releases** [Array, required]: The name and version of each release in the deployment. -* **name** [String, required]: Name of a release used in the deployment. -* **version** [String, required]: The version of the release to use. Version can be `latest`. +- **name** [String, required]: Name of a release used in the deployment. +- **version** [String, required]: The version of the release to use. Version can be `latest`. Example: @@ -51,9 +55,9 @@ releases: **releases** [Array, required]: The name, url and possibly SHA1 of each release in the deployment. -* **name** [String, required]: Name of a release used in the deployment. -* **url** [String, required]: URL of the release to use. URL may use the file protocol (`file://`) or HTTP(s) (`http(s)://`). File URLs can be absolute or relative to the current directory of `bosh-init` execution. -* **sha1** [String, required]: The SHA1 of the release tarball. SHA1 is only required when using HTTP(s) URLs. +- **name** [String, required]: Name of a release used in the deployment. +- **url** [String, required]: URL of the release to use. URL may use the file protocol (`file://`) or HTTP(s) (`http(s)://`). File URLs can be absolute or relative to the current directory of `bosh-init` execution. +- **sha1** [String, required]: The SHA1 of the release tarball. SHA1 is only required when using HTTP(s) URLs. Example: @@ -70,9 +74,9 @@ releases: **releases** [Array, required]: The name and local release directory of a release in the deployment. -* **name** [String, required]: Name of a release used in the deployment. -* **url** [String, required]: Path to release directory on local filesystem (relative to working directory). -* **version** [String, required]: Must be `create` +- **name** [String, required]: Name of a release used in the deployment. +- **url** [String, required]: Path to release directory on local filesystem (relative to working directory). +- **version** [String, required]: Must be `create` Example: @@ -84,6 +88,7 @@ releases: ``` --- + ## Networks Block {: #networks } **networks** [Array, required]: Each sub-block listed in the Networks block specifies a network configuration that jobs can reference. There are three different network types: `manual`, `dynamic`, and `vip`. @@ -99,18 +104,19 @@ See [networks](networks.md) for more details. - [See vSphere CPI network cloud properties](vsphere-cpi.md#networks) --- + ## Resource Pools Block {: #resource-pools } **resource_pools** [Array, required]: Specifies the [resource pools](terminology.md#resource-pool) a deployment uses. A deployment manifest can describe multiple resource pools and uses unique names to identify and reference them. -* **name** [String, required]: A unique name used to identify and reference the resource pool -* **network** [String, required]: References a valid network name defined in the Networks block. Newly created resource pool VMs use the described configuration. -* **size** [Integer, optional]: The number of VMs in the resource pool. If you omit this value, BOSH calculates the resource pool size based on the total number of job instances that belong to this resource pool. If you specify this value, BOSH requires that the size be at least as large as the total number of job instances using it. -* **stemcell** [Hash, required]: The stemcell used to create resource pool VMs. - * **name** [String, required]: The stemcell name - * **version** [String, required]: The stemcell version -* **cloud_properties** [Hash, optional]: Describes any IaaS-specific properties needed to create VMs. Most IaaSes require this. See [CPI Specific `cloud_properties`](#resource-pools-cloud-properties) below. Examples: `instance_type`, `availability_zone`. Default is `{}` (empty Hash). -* **env** [Hash, optional]: Describes the VM environment and provides a specific VM environment to the CPI `create_stemcell` call. `env` data is available to BOSH Agents as VM settings. Default is `{}` (empty Hash). +- **name** [String, required]: A unique name used to identify and reference the resource pool +- **network** [String, required]: References a valid network name defined in the Networks block. Newly created resource pool VMs use the described configuration. +- **size** [Integer, optional]: The number of VMs in the resource pool. If you omit this value, BOSH calculates the resource pool size based on the total number of job instances that belong to this resource pool. If you specify this value, BOSH requires that the size be at least as large as the total number of job instances using it. +- **stemcell** [Hash, required]: The stemcell used to create resource pool VMs. + - **name** [String, required]: The stemcell name + - **version** [String, required]: The stemcell version +- **cloud_properties** [Hash, optional]: Describes any IaaS-specific properties needed to create VMs. Most IaaSes require this. See [CPI Specific `cloud_properties`](#resource-pools-cloud-properties) below. Examples: `instance_type`, `availability_zone`. Default is `{}` (empty Hash). +- **env** [Hash, optional]: Describes the VM environment and provides a specific VM environment to the CPI `create_stemcell` call. `env` data is available to BOSH Agents as VM settings. Default is `{}` (empty Hash). Example: @@ -130,8 +136,8 @@ resource_pools: **stemcell** [Hash, required]: The stemcell used to create resource pool VMs. - - **url** [String, required]: URL of the stemcell tarball. URL may use the file protocol (`file://`) or HTTP(s) (`http(s)://`). File URLs can be absolute or relative to the current directory of `bosh-init` execution. - - **sha1** [String, required]: The SHA1 of the stemcell tarball. SHA1 is only required when using HTTP(s) URLs. +- **url** [String, required]: URL of the stemcell tarball. URL may use the file protocol (`file://`) or HTTP(s) (`http(s)://`). File URLs can be absolute or relative to the current directory of `bosh-init` execution. +- **sha1** [String, required]: The SHA1 of the stemcell tarball. SHA1 is only required when using HTTP(s) URLs. Example: @@ -156,13 +162,14 @@ resource_pools: - [See vSphere CPI resource pool cloud properties](vsphere-cpi.md#resource-pools) --- + ## Disk Pools Block {: #disk-pools } **disk_pools** [Array, required]: Specifies the [disk pools](terminology.md#disk-pool) a deployment uses. A deployment manifest can describe multiple disk pools and uses unique names to identify and reference them. -* **name** [String, required]: A unique name used to identify and reference the disk pool -* **disk_size** [Integer, required]: Specifies the disk size. `disk_size` must be a positive integer. BOSH creates a [persistent disk](persistent-disks.md) of that size in megabytes and attaches it to each job instance VM. -* **cloud_properties** [Hash, optional]: Describes any IaaS-specific properties needed to create disks. Examples: `type`, `iops`. Default is `{}` (empty Hash). +- **name** [String, required]: A unique name used to identify and reference the disk pool +- **disk_size** [Integer, required]: Specifies the disk size. `disk_size` must be a positive integer. BOSH creates a [persistent disk](persistent-disks.md) of that size in megabytes and attaches it to each job instance VM. +- **cloud_properties** [Hash, optional]: Describes any IaaS-specific properties needed to create disks. Examples: `type`, `iops`. Default is `{}` (empty Hash). Example: @@ -183,14 +190,15 @@ disk_pools: - [See vSphere CPI disk pool cloud properties](vsphere-cpi.md#disk-pools) --- + ## Compilation Block {: #compilation } **compilation** [Hash, required]: Properties of compilation VMs. -* **workers** [Integer, required]: The maximum number of compilation VMs. -* **network** [String, required]: References a valid network name defined in the Networks block. BOSH assigns network properties to compilation VMs according to the type and properties of the specified network. -* **reuse\_compilation\_vms** [Boolean, optional]: If `false`, BOSH creates a new compilation VM for each new package compilation and destroys the VM when compilation is complete. If `true`, compilation VMs are re-used when compiling packages. Defaults to `false`. -* **cloud_properties** [Hash, optional]: Describes any IaaS-specific properties needed to create compilation VMs; for most IaaSes, some data here is actually required. For the Compilation Block, the required `cloud_properties` are the same as for Resource Pools; see the [CPI Specific `cloud_properties`](#resource-pools-cloud-properties) for Resource Pools. Examples: `instance_type`, `availability_zone`. Default is `{}` (empty Hash). +- **workers** [Integer, required]: The maximum number of compilation VMs. +- **network** [String, required]: References a valid network name defined in the Networks block. BOSH assigns network properties to compilation VMs according to the type and properties of the specified network. +- **reuse\_compilation\_vms** [Boolean, optional]: If `false`, BOSH creates a new compilation VM for each new package compilation and destroys the VM when compilation is complete. If `true`, compilation VMs are re-used when compiling packages. Defaults to `false`. +- **cloud_properties** [Hash, optional]: Describes any IaaS-specific properties needed to create compilation VMs; for most IaaSes, some data here is actually required. For the Compilation Block, the required `cloud_properties` are the same as for Resource Pools; see the [CPI Specific `cloud_properties`](#resource-pools-cloud-properties) for Resource Pools. Examples: `instance_type`, `availability_zone`. Default is `{}` (empty Hash). Example: @@ -205,24 +213,25 @@ compilation: ``` --- + ## Update Block {: #update } **update** [Hash, required]: This specifies instance update properties. These properties control how BOSH updates job instances during the deployment. -* **canaries** [Integer or Percentage, required]: The number of [canary](terminology.md#canary) instances. -* **max\_in\_flight** [Integer or Percentage, required]: The maximum number of non-canary instances to update in parallel within an availability zone. Updates will not begin in another availability zone until all VMs are updated in the current availability zone. - * If the `max_in_flight` is a percentage, the minimum `max_in_flight` will never fall below 1. -* **canary\_watch\_time** [Integer or Range, required] - * If the `canary_watch_time` is an integer, the Director sleeps for that many milliseconds, then checks whether the canary instances are healthy. - * If the `canary_watch_time` is a range (low-high), the Director: - * Waits for `low` milliseconds - * Waits until instances are healthy or `high` milliseconds have passed since instances started updating -* **update\_watch\_time** [Integer or Range, required] - * If the `update_watch_time` is an integer, the Director sleeps for that many milliseconds, then checks whether the instances are healthy. - * If the `update_watch_time` is a range (low-high), the Director: - * Waits for `low` milliseconds - * Waits until instances are healthy or `high` milliseconds have passed since instances started updating -* **serial** [Boolean, optional]: If disabled (set to `false`), deployment jobs will be deployed in parallel, otherwise - sequentially. Instances within a deployment job will still follow `canary` and `max_in_flight` configuration. Defaults to `true`. +- **canaries** [Integer or Percentage, required]: The number of [canary](terminology.md#canary) instances. +- **max\_in\_flight** [Integer or Percentage, required]: The maximum number of non-canary instances to update in parallel within an availability zone. Updates will not begin in another availability zone until all VMs are updated in the current availability zone. + - If the `max_in_flight` is a percentage, the minimum `max_in_flight` will never fall below 1. +- **canary\_watch\_time** [Integer or Range, required] + - If the `canary_watch_time` is an integer, the Director sleeps for that many milliseconds, then checks whether the canary instances are healthy. + - If the `canary_watch_time` is a range (low-high), the Director: + - Waits for `low` milliseconds + - Waits until instances are healthy or `high` milliseconds have passed since instances started updating +- **update\_watch\_time** [Integer or Range, required] + - If the `update_watch_time` is an integer, the Director sleeps for that many milliseconds, then checks whether the instances are healthy. + - If the `update_watch_time` is a range (low-high), the Director: + - Waits for `low` milliseconds + - Waits until instances are healthy or `high` milliseconds have passed since instances started updating +- **serial** [Boolean, optional]: If disabled (set to `false`), deployment jobs will be deployed in parallel, otherwise - sequentially. Instances within a deployment job will still follow `canary` and `max_in_flight` configuration. Defaults to `true`. Examples: @@ -244,24 +253,25 @@ update: ``` --- + ## Jobs Block {: #jobs } **jobs** [Array, required]: Specifies the mapping between BOSH release [jobs](terminology.md#job) and cloud instances. Jobs are defined in the BOSH release. The Jobs block defines how BOSH associates jobs with the VMs started by the IaaS. The most commonly used job properties are: -* **name** [String, required]: A unique name used to identify and reference this association between a BOSH release job and a VM. -* **templates** [Array, required]: Specifies the name and release of a job template. - * **name** [String, required]: The job template name - * **release** [String, required]: The release where the job template exists -* **lifecycle** [String, optional]: Specifies the kind of task the job represents. Valid values are `service` and `errand`; defaults to `service`. A `service` runs indefinitely and restarts if it fails. An `errand` starts with a manual trigger and does not restart if it fails. -* **persistent_disk** [Integer, optional]: Specifies the persistent disk size; defaults to 0 (no persistent disk). If `persistent_disk` is a positive integer, BOSH creates a persistent disk of that size in megabytes and attaches it to each job instance VM. [Read more about persistent disks](persistent-disks.md) -* **properties** [Hash, optional]: Specifies job properties. Properties allow BOSH to configure jobs to a specific environment. `properties` defined in a Job block are accessible only to that job, and override any identically named global properties. -* **resource_pool** [String, required]: A valid resource pool name from the Resource Pools block. BOSH runs instances of this job in a VM from the named resource pool. -* **update** [Hash, optional]: Specific update settings for this job. Use this to override [global job update settings](#update) on a per-job basis. -* **instances** [Integer, required]: The number of job instances. Each instance is a VM running this particular job. -* **networks** [Array, required]: Specifies the networks this job requires. Each network can have the following properties specified: - * **name** [String, required]: A valid network name from the Networks block - * **static_ips** [Array, optional]: Array of IP addresses reserved for the job on the network - * **default** [Array, optional]: Specifies which network components (DNS, Gateway) BOSH populates by default from this network. BOSH references this property if the Networks block defines multiple networks. +- **name** [String, required]: A unique name used to identify and reference this association between a BOSH release job and a VM. +- **templates** [Array, required]: Specifies the name and release of a job template. + - **name** [String, required]: The job template name + - **release** [String, required]: The release where the job template exists +- **lifecycle** [String, optional]: Specifies the kind of task the job represents. Valid values are `service` and `errand`; defaults to `service`. A `service` runs indefinitely and restarts if it fails. An `errand` starts with a manual trigger and does not restart if it fails. +- **persistent_disk** [Integer, optional]: Specifies the persistent disk size; defaults to 0 (no persistent disk). If `persistent_disk` is a positive integer, BOSH creates a persistent disk of that size in megabytes and attaches it to each job instance VM. [Read more about persistent disks](persistent-disks.md) +- **properties** [Hash, optional]: Specifies job properties. Properties allow BOSH to configure jobs to a specific environment. `properties` defined in a Job block are accessible only to that job, and override any identically named global properties. +- **resource_pool** [String, required]: A valid resource pool name from the Resource Pools block. BOSH runs instances of this job in a VM from the named resource pool. +- **update** [Hash, optional]: Specific update settings for this job. Use this to override [global job update settings](#update) on a per-job basis. +- **instances** [Integer, required]: The number of job instances. Each instance is a VM running this particular job. +- **networks** [Array, required]: Specifies the networks this job requires. Each network can have the following properties specified: + - **name** [String, required]: A valid network name from the Networks block + - **static_ips** [Array, optional]: Array of IP addresses reserved for the job on the network + - **default** [Array, optional]: Specifies which network components (DNS, Gateway) BOSH populates by default from this network. BOSH references this property if the Networks block defines multiple networks. Example: @@ -287,6 +297,7 @@ jobs: ``` --- + ## Properties Block {: #properties } **properties** [Hash, optional]: Describes global properties and general configuration information. @@ -297,12 +308,12 @@ By default, general configuration information resides in the template spec file Typical general configuration information includes but is not limited to: -* Passwords -* Account names -* Shared secrets -* Host names -* IP addresses -* Port numbers +- Passwords +- Account names +- Shared secrets +- Host names +- IP addresses +- Port numbers Example: @@ -322,15 +333,16 @@ properties: If you declare specific properties in a job template spec, BOSH ignores all other properties. If you do not declare any specific properties in a job template spec, BOSH applies all properties from the deployment manifest to the job. --- + ## Cloud Provider Block {: #cloud-provider } **cloud_provider** [Hash, required]: Specifies CPI configuration for the `bosh-init` to create VMs, etc. Regular deployment manifests cannot specify this block. -* **template** [Hash, required]: Specifies the name of the CPI job and release where the CPI job exists. It will be used by the `bosh-init` to create VMs, persistent disks, etc. - * **name** [String, required]: The CPI job name. - * **release** [String, required]: The CPI release name. -* **mbus** [String, required]: HTTPs URL used by the `bosh-init` to contact the Agent on a created VM. The URL includes basic auth credentials that should be customized for each deployment. Example: `"https://mbus:mbus-password@10.0.0.6:6868"`. -* **properties** [Hash, required]: Properties required by the CPI job. +- **template** [Hash, required]: Specifies the name of the CPI job and release where the CPI job exists. It will be used by the `bosh-init` to create VMs, persistent disks, etc. + - **name** [String, required]: The CPI job name. + - **release** [String, required]: The CPI release name. +- **mbus** [String, required]: HTTPs URL used by the `bosh-init` to contact the Agent on a created VM. The URL includes basic auth credentials that should be customized for each deployment. Example: `"https://mbus:mbus-password@10.0.0.6:6868"`. +- **properties** [Hash, required]: Properties required by the CPI job. Example from [Initializing BOSH environment on vSphere](init-vsphere.md): diff --git a/content/deployment.md b/content/deployment.md index 93899d551..eb746c9a1 100644 --- a/content/deployment.md +++ b/content/deployment.md @@ -1,3 +1,5 @@ +# What is a Deployment? + A deployment is a collection of VMs, built from a [stemcell](stemcell.md), that has been populated with specific [releases](release.md) and disks that keep persistent data. These resources are created in the IaaS based on a deployment manifest and managed by the [Director](terminology.md#director), a centralized management server. The deployment process begins with deciding which Operating System images (stemcells) need to be used and which software (releases) need to be deployed, how to track persistent data while a cluster is updated and transformed, and how to automate steps of deploying images to an IaaS; this includes configuring machines to use said image, and placing correct software versions onto provisioned machines. BOSH builds upon previously introduced primitives (stemcells and releases) by providing a way to state an explicit combination of stemcells, releases, and operator-specified properties in a human readable file. This file is called a deployment manifest. diff --git a/content/design.md b/content/design.md index 08f647d92..cca6e046e 100644 --- a/content/design.md +++ b/content/design.md @@ -1,13 +1,12 @@ +# Design + BOSH is an open framework for managing the full development and deployment life cycle of large-scale distributed software applications. BOSH: -* Leverages IaaS APIs to create VMs from base images packaged with - operator-defined network, storage, and software configurations -* Monitors and manages VM and process health, detecting and restarting processes - or VMs when they become unhealthy -* Updates all VMs reliably and idempotently, whether the update is to the OS, a - package, or any other component +- Leverages IaaS APIs to create VMs from base images packaged with operator-defined network, storage, and software configurations +- Monitors and manages VM and process health, detecting and restarting processes or VMs when they become unhealthy +- Updates all VMs reliably and idempotently, whether the update is to the OS, a package, or any other component ## BOSH Deployments are Predictable {: #predictable } diff --git a/content/director-access-events.md b/content/director-access-events.md index 541784372..edf802b87 100644 --- a/content/director-access-events.md +++ b/content/director-access-events.md @@ -1,3 +1,5 @@ +# Logging API Access + !!! note This feature is available in bosh-release v256+. @@ -5,13 +7,13 @@ Director logs all API access events to syslog under `vcap.bosh.director` topic. Here is a log snipped found in `/var/log/syslog` in [Common Event Format (CEF)](https://www.protect724.hpe.com/servlet/JiveServlet/downloadBody/1072-102-7-18874/CommonEventFormat%20v22.pdf): -``` +```shell May 13 05:13:34 localhost vcap.bosh.director[16199]: CEF:0|CloudFoundry|BOSH|1.0000.0|director_api|/deployments|7|requestMethod=GET src=127.0.0.1 spt=25556 shost=36ff45a2-51a2-488d-af95-953c43de4cec cs1=10.10.0.36,fe80::80a:99ff:fed6:df7d%eth0 cs1Label=ips cs2=X_BOSH_UPLOAD_REQUEST_TIME=0.000&HOST=127.0.0.1&X_REAL_IP=127.0.0.1&X_FORWARDED_FOR=127.0.0.1&X_FORWARDED_PROTO=https&USER_AGENT=EventMachine HttpClient cs2Label=httpHeaders cs3=none cs3Label=authType cs4=401 cs4Label=responseStatus cs5=Not authorized: '/deployments' cs5Label=statusReason ``` And in a more readable form: -``` +```shell May 13 05:13:34 localhost vcap.bosh.director[16199]: CEF:0 CloudFoundry @@ -43,6 +45,7 @@ cs5Label=statusReason ``` --- + ## Enabling Logging {: #enable } To enable this feature: diff --git a/content/director-api-v1.md b/content/director-api-v1.md index 5090dc948..13d9f4eb0 100644 --- a/content/director-api-v1.md +++ b/content/director-api-v1.md @@ -1,9 +1,12 @@ +# Director HTTP API + !!! note Before using the Director API directly, we strongly encourage to consider using the CLI for automation such as performing a scheduled deploy from a CI. We hope that you will open a [GitHub issue](https://github.com/cloudfoundry/bosh/issues) to share your use cases so that we can suggest or possibly make additions to the CLI. This document lists common API endpoints provided by the Director. --- + ## Overview {: #overview } ### Discovery through bosh cli {: #cli-debug } @@ -19,10 +22,11 @@ BOSH_LOG_LEVEL=debug bosh cpi-config |& grep 'request to endpoint' ### Discovery through source code {: #source-code } As a complement to this reference documentation, reading the bosh director source code can be useful to discover the latest endpoints and their supported options: -* [controllers specs/unit tests](https://github.com/cloudfoundry/bosh/blob/main/src/bosh-director/spec/unit/api/controllers) -* [controllers](https://github.com/cloudfoundry/bosh/tree/main/src/bosh-director/lib/bosh/director/api/controllers) -Below is a sample rendering of the deployment controller specs in intellij IDE with ruby support enabled: +- [controllers specs/unit tests](https://github.com/cloudfoundry/bosh/blob/main/src/bosh-director/spec/unit/api/controllers) +- [controllers](https://github.com/cloudfoundry/bosh/tree/main/src/bosh-director/lib/bosh/director/api/controllers) + +Below is a sample rendering of the deployment controller specs in intellij IDE with ruby support enabled: ![Sample director api specs.png](images/director-controller-unit-test-specs.png) ### Security {: #auth } @@ -39,11 +43,11 @@ The [`bosh curl`](cli-v2.md#curl) command makes authenticated http requests to t Standard HTTP verb semantics are followed: -| Verb | Description | --------|-------------| -| GET | Used for retrieving resources. | +| Verb | Description | +|----------|---------------------------------------| +| GET | Used for retrieving resources. | | POST/PUT | Used for creating/updating resources. | -| DELETE | Used for deleting resources. | +| DELETE | Used for deleting resources. | ### HTTP redirects {: #http-verbs } @@ -64,6 +68,7 @@ Certain requests result in complex and potentially long running operations again Once a Director task is created, clients can follow its progress by polling [`GET /tasks/{id}`](#get-task) to find out its state. While waiting for the task to finish, different types of logs ([event](#get-task-event), [result](#get-task-result), [debug](#get-task-debug) information, etc.) can be followed to gain insight into what the task is doing. --- + ## General {: #general } ### `GET /info`: Info {: #info } @@ -154,6 +159,7 @@ bosh curl /configs?latest=true | jq . ``` --- + ### `POST /configs`: Create config. {: #create-config } #### Request headers @@ -194,6 +200,7 @@ curl -s -k -H 'Content-Type: application/json' -d '{"name": "test", "type": "clo ``` --- + ### `POST /configs/diff`: Diff config. {: #diff-config } #### Request headers @@ -233,6 +240,7 @@ curl -s -k -H 'Content-Type: application/json' -d '{"name": "default", "type": " ``` --- + ### `DELETE /configs`: Marks configs as deleted. {: #delete-config } #### Request Query @@ -247,6 +255,7 @@ curl -s -k -X DELETE https://192.168.56.4:25555/configs?type=cloud&name=test ``` --- + ## Tasks {: #tasks } See [Director tasks](director-tasks.md) for related info. @@ -369,6 +378,7 @@ bosh curl '/tasks?context_id=4528' | jq . ``` --- + ### `GET /tasks/{id}`: Retrieve single task {: #get-task } #### Response body schema @@ -397,6 +407,7 @@ bosh curl '/tasks/1180' | jq . ``` --- + ### `GET /tasks/{id}/output?type=debug`: Retrieve task's debug log {: #get-task-debug } #### Response body schema @@ -418,6 +429,7 @@ D, [2015-11-09 02:19:36 #32545] [task:1180] DEBUG -- DirectorJobRunner: (0.00031 ``` --- + ### `GET /tasks/{id}/output?type=event`: Retrieve task's event log {: #get-task-event } #### Response body schema @@ -457,6 +469,7 @@ bosh curl '/tasks/1181/output?type=event' ``` --- + ### `GET /tasks/{id}/output?type=result`: Retrieve task's result {: #get-task-result } #### Response body schema @@ -472,56 +485,61 @@ bosh curl '/tasks/1181/output?type=event' ```json { - "vm_cid": "ec974048-3352-4ba4-669d-beab87b16bcb", - "disk_cid": null, - "ips": ["10.244.0.142"], - "dns": [], - "agent_id": "c5e7c705-459e-41c0-b640-db32d8dc6e71", - "job_name": "doppler_z1", - "index": 0, - "job_state": "running", - "resource_pool": "medium_z1", - "vitals": { - "cpu": { - "sys": "9.1", - "user": "2.1", - "wait": "1.7" - }, - "disk": { - "ephemeral": { - "inode_percent": "11", - "percent": "36" - }, - "system": { - "inode_percent": "11", - "percent": "36" - } - }, - "load": ["0.61", "0.74", "1.10"], - "mem": { - "kb": "2520960", - "percent": "41" - }, - "swap": { - "kb": "102200", - "percent": "10" - } - }, - "processes": [{ - "name": "doppler", - "state": "running" - }, { - "name": "syslog_drain_binder", - "state": "running" - }, { - "name": "metron_agent", - "state": "running" - }] + "vm_cid": "ec974048-3352-4ba4-669d-beab87b16bcb", + "disk_cid": null, + "ips": ["10.244.0.142"], + "dns": [], + "agent_id": "c5e7c705-459e-41c0-b640-db32d8dc6e71", + "job_name": "doppler_z1", + "index": 0, + "job_state": "running", + "resource_pool": "medium_z1", + "vitals": { + "cpu": { + "sys": "9.1", + "user": "2.1", + "wait": "1.7" + }, + "disk": { + "ephemeral": { + "inode_percent": "11", + "percent": "36" + }, + "system": { + "inode_percent": "11", + "percent": "36" + } + }, + "load": ["0.61", "0.74", "1.10"], + "mem": { + "kb": "2520960", + "percent": "41" + }, + "swap": { + "kb": "102200", + "percent": "10" + } + }, + "processes": [ + { + "name": "doppler", + "state": "running" + }, + { + "name": "syslog_drain_binder", + "state": "running" + }, + { + "name": "metron_agent", + "state": "running" + } + ] } ``` --- + ### `POST /tasks/cancel`: Cancel tasks {: #cancel-tasks } #### Request headers @@ -543,6 +561,7 @@ Empty. --- + ## Stemcells {: #stemcells } ### `GET /stemcells`: List all uploaded stemcells {: #list-stemcells } @@ -580,6 +599,7 @@ bosh curl '/stemcells' | jq . ``` --- + ## Releases {: #releases } ### `GET /releases`: List all uploaded releases {: #list-releases } @@ -640,6 +660,7 @@ bosh curl '/releases' | jq . ``` --- + ## Deployments {: #deployments } ### `GET /deployments`: List all deployments {: #list-deployments } @@ -694,6 +715,7 @@ bosh curl '/deployments' | jq . ``` --- + ### `GET /deployments?exclude_configs=true&exclude_releases=true&exclude_stemcells=true`: List all deployments without configs, releases, and stemcells {: #list-just-deployments } #### Response body schema @@ -718,6 +740,7 @@ bosh curl '/deployments?exclude_configs=true&exclude_releases=true&exclude_stemc ``` --- + ### `POST /deployments`: Create/update single deployment {: #post-deployment } #### Request query @@ -739,6 +762,7 @@ bosh curl '/deployments?exclude_configs=true&exclude_releases=true&exclude_stemc Creating/updating a deployment is performed in a Director task. Response will be a redirect to a task resource. --- + ### `GET /deployments/{name}`: Retrieve single deployment {: #get-deployment } #### Response body schema @@ -761,6 +785,7 @@ bosh curl '/deployments/cf-warden' | jq . ``` --- + ### `DELETE /deployments/{name}`: Delete single deployment {: #delete-deployment } #### Request query @@ -776,6 +801,7 @@ Empty. Deleting a deployment is performed in a Director task. Response will be a redirect to a task resource. --- + ## Instances in a deployment {: #instances } !!! note @@ -829,6 +855,7 @@ bosh curl '/deployments/example/instances' | jq . ``` --- + ### `GET /deployments/{name}/instances?format=full`: List details of instances {: #list-instances-detailed } #### Response body schema @@ -954,6 +981,7 @@ curl -v -s -k 'https://admin:admin@192.168.56.4:25555/tasks/1287/output?type=res ``` --- + ## VMs in a deployment {: #vms } ### `GET /deployments/{name}/vms`: List all VMs {: #list-vms } @@ -1018,6 +1046,7 @@ bosh curl '/deployments/cf-warden/vms' | jq . ``` --- + ### `GET /deployments/{name}/vms?format=full`: List VM details {: #list-vms-detailed } #### Response body schema @@ -1066,104 +1095,104 @@ curl -v -s -k 'https://admin:admin@192.168.56.4:25555/tasks/1181/output?type=res ```json { - "vm_cid": "3938cc70-8f5e-4318-ad05-24d991e0e66e", - "active": true, - "vm_created_at": "2023-01-11T14:11:02Z", - "cloud_properties": { - "zone": "us-east1-c", - "machine_type": "n1-standard-2", - "root_disk_size_gb": 20, - "root_disk_type": "pd-ssd" - }, - "disk_cid": null, - "ips": ["10.0.1.3"], - "dns": [], - "agent_id": "d927e75b-2a2d-4015-b5cc-306a067e94e9", - "job_name": "example_service", - "index": 0, - "job_state": "running", - "state": "started", - "resource_pool": "resource_pool_1", - "vm_type": "resource_pool_1", - "vitals": { - "cpu": { - "sys": "0.3", - "user": "0.1", - "wait": "0.0" - }, - "disk": { - "ephemeral": { - "inode_percent": "5", - "percent": "32" - }, - "persistent": { - "inode_percent": "3", - "percent": "67" - }, - "system": { - "inode_percent": "34", - "percent": "66" - } - }, - "load": ["0.00", "0.01", "0.10"], - "mem": { - "kb": "605008", - "percent": "7" - }, - "swap": { - "kb": "75436", - "percent": "1" - } - }, + "vm_cid": "3938cc70-8f5e-4318-ad05-24d991e0e66e", + "active": true, + "vm_created_at": "2023-01-11T14:11:02Z", + "cloud_properties": { + "zone": "us-east1-c", + "machine_type": "n1-standard-2", + "root_disk_size_gb": 20, + "root_disk_type": "pd-ssd" + }, + "disk_cid": null, + "ips": ["10.0.1.3"], + "dns": [], + "agent_id": "d927e75b-2a2d-4015-b5cc-306a067e94e9", + "job_name": "example_service", + "index": 0, + "job_state": "running", + "state": "started", + "resource_pool": "resource_pool_1", + "vm_type": "resource_pool_1", + "vitals": { + "cpu": { + "sys": "0.3", + "user": "0.1", + "wait": "0.0" + }, + "disk": { + "ephemeral": { + "inode_percent": "5", + "percent": "32" + }, + "persistent": { + "inode_percent": "3", + "percent": "67" + }, + "system": { + "inode_percent": "34", + "percent": "66" + } + }, + "load": ["0.00", "0.01", "0.10"], + "mem": { + "kb": "605008", + "percent": "7" + }, + "swap": { + "kb": "75436", + "percent": "1" + } + }, "processes": [{ - "name": "beacon", - "state": "running", - "uptime": { - "secs": 1212184 - }, - "mem": { - "kb": 776, - "percent": 0 - }, - "cpu": { - "total": 0 - } + "name": "beacon", + "state": "running", + "uptime": { + "secs": 1212184 + }, + "mem": { + "kb": 776, + "percent": 0 + }, + "cpu": { + "total": 0 + } }, { - "name": "baggageclaim", - "state": "running", - "uptime": { - "secs": 1212152 - }, - "mem": { - "kb": 8920, - "percent": 0.1 - }, - "cpu": { - "total": 0 - } + "name": "baggageclaim", + "state": "running", + "uptime": { + "secs": 1212152 + }, + "mem": { + "kb": 8920, + "percent": 0.1 + }, + "cpu": { + "total": 0 + } }, { - "name": "garden", - "state": "running", - "uptime": { - "secs": 1212153 - }, - "mem": { - "kb": 235004, - "percent": 2.8 - }, - "cpu": { - "total": 0.2 - } + "name": "garden", + "state": "running", + "uptime": { + "secs": 1212153 + }, + "mem": { + "kb": 235004, + "percent": 2.8 + }, + "cpu": { + "total": 0.2 + } }], - "az": null, - "id": "abe6a4e9-cfca-490b-8515-2893f9e54d20", - "bootstrap": false, - "ignore": false, - "stemcell": { - "name": "bosh-google-kvm-ubuntu-xenial-go_agent", - "version": "621.359", - "api_version": 3 - } + "az": null, + "id": "abe6a4e9-cfca-490b-8515-2893f9e54d20", + "bootstrap": false, + "ignore": false, + "stemcell": { + "name": "bosh-google-kvm-ubuntu-xenial-go_agent", + "version": "621.359", + "api_version": 3 + } } ``` @@ -1229,6 +1258,7 @@ curl -v -s -k 'https://admin:admin@192.168.56.4:25555/tasks/1181/output?type=res ``` --- + ## Events {: #events } See [Events](events.md) for info. @@ -1323,6 +1353,7 @@ bosh curl /events | jq . ``` --- + ### `GET /events/{id}`: Retrieve single event {: #get-event } #### Response body schema @@ -1355,6 +1386,7 @@ bosh curl /events/3133 | jq . ``` --- + ### `POST /events`: Create single event {: #post-event } #### Request body schema diff --git a/content/director-backup.md b/content/director-backup.md index 5c43df88a..69d6becaa 100644 --- a/content/director-backup.md +++ b/content/director-backup.md @@ -1,3 +1,5 @@ +# CLI v1 Backup / Restore + ## Why performing bosh director backup and restores ? {: #why-backup } If using bosh-init to deploy your bosh-director, it is useful to backup the deployment state file containing associated Iaas information (IP, floating IP, persistent disk volume id). This would enable recovery of a lost bosh director VM from the persistent disk still present in the Iaas. See [recovering state](cli-envs.md#recover-deployment-state). @@ -10,11 +12,15 @@ Bosh provides built-in commands to export the content of the director database a Target the bosh director deployment that you need to backup: -`bosh deployment your_bosh_director_manifest` +```shell +bosh deployment your_bosh_director_manifest +``` -Make a backup of your bosh instance : +Make a backup of your bosh instance: -`bosh backup` +```shell +bosh backup +``` This command will generate a .tar.gz file that contains a dump of director's database @@ -26,27 +32,34 @@ The first step is to deploy a new fresh empty bosh director deployment. The second step is to restore the content of the bosh director db -* Connect to your bosh instance (using `bosh target https://your_bosh_ip`). -* Use the bosh restore command : `bosh restore yourBackupFile.tar.gz` +- Connect to your bosh instance (using `bosh target https://your_bosh_ip`). +- Use the bosh restore command : `bosh restore yourBackupFile.tar.gz` The third step is to manually re-upload the releases and stemcells binaries. -* List the expected stemcells and releases +- List the expected stemcells and releases - `bosh stemcells` - `bosh releases` +```shell +bosh stemcells +bosh releases +``` Then upload stemcells/releases from your repositories such as bosh.io, using the `--fix` option so bosh will fix them in the db (fixing the missing blobs). Without the `--fix` option, bosh would complain about duplicate stemcells/releases with same name. - `bosh upload stemcell https://stemcells_url --fix` - `bosh upload releases https://release_url --fix` +```shell +bosh upload stemcell https://stemcells_url --fix +bosh upload releases https://release_url --fix +``` Following these restoration steps, the bosh director is now able to manage your previous deployments. -`bosh cloudcheck` - -`bosh instances --ps` +```shell +bosh cloudcheck +bosh instances --ps +``` You can now safely update your deployments using the usual deploy command. -`bosh deploy` +```shell +bosh deploy +``` diff --git a/content/director-blobstore-signed-urls.md b/content/director-blobstore-signed-urls.md index f426119e9..7b0d0231e 100644 --- a/content/director-blobstore-signed-urls.md +++ b/content/director-blobstore-signed-urls.md @@ -1,3 +1,5 @@ +# Bosh Director Blobstore Signed URLs + !!! note "Version compatibility" This `blobstore.enable_signed_urls` config property was first introduced in bosh v270.8, the ubuntu-xenial 621.x stemcell, and the windows 2019.17 stemcell. @@ -17,7 +19,7 @@ later, and windows 2019.17 and later. This feature can be enabled by updating the Bosh director manifest with the following properties: -* `blobstore.enable_signed_urls`: set this to `true` to have the director +- `blobstore.enable_signed_urls`: set this to `true` to have the director begin sending signed urls to the agent. Enabling signed URLs should work alongside blobstore provider specific @@ -43,41 +45,38 @@ credentials. As an operator, you can remove credentials from deployment VMs by making the following changes: -* If all deployments are using supported stemcells, override - the `blobstores` to an empty array in the director manifest. - - ``` - instance_groups: - - name: bosh - ... - properties: - agent: +- If all deployments are using supported stemcells, override the `blobstores` to an empty array in the director manifest. + + ```yaml + instance_groups: + - name: bosh + ... + properties: + agent: + env: + bosh: + blobstores: [] + ``` + +- For deployments on supported stemcells, override individual deployment manifests with the following property: + + ```yaml + instance_groups: + - name: zookeeper + ... + properties: env: bosh: blobstores: [] - ``` - -* For deployments on supported stemcells, override individual deployment manifests with the following - property: + ``` - ``` - instance_groups: - - name: zookeeper - ... - properties: - env: - bosh: - blobstores: [] - ``` +- For deployments on unsupported stemcells, please do not make any blobstore modifications as blobstore config may be coming from `blobstores` or your CPI. -* For deployments on unsupported stemcells, please do not make any blobstore - modifications as blobstore config may be coming from `blobstores` or your CPI. - -**DAV ONLY** +### DAV ONLY For DAV blobstores, please also configure: -* `blobstore.secret`: The secret used to calculate the signed urls' signature +- `blobstore.secret`: The secret used to calculate the signed urls' signature ## Notes @@ -87,4 +86,3 @@ update the property to false, you **must** also recreate all VMs managed by bosh in order to propagate blobstore credentials to the VMs. If you do not recreate the VMs, none of the agents will have blobstore credentials to correctly process requests. - diff --git a/content/director-bosh-teams.md b/content/director-bosh-teams.md index c82ea714e..1ee4f3f7f 100644 --- a/content/director-bosh-teams.md +++ b/content/director-bosh-teams.md @@ -1,3 +1,5 @@ +# Using BOSH Teams + BOSH Teams map to UAA scopes that restrict the set of deployments that a user can manage. A user's or client's membership in a BOSH team is determined by the scopes assigned to their UAA client. @@ -14,13 +16,13 @@ Scopes can be added to existing clients in order to associate the clients with BOSH Teams. You must be logged into UAA as [a privileged user to grant and revoke scopes](director-users-uaa-scopes.md#user-login). All BOSH Team scopes follow the format: -``` +```text bosh.teams.. ``` To add a BOSH Team scope to an existing client: -``` +```shell uaac client update --authorities bosh.teams..admin ``` diff --git a/content/director-certs-openssl.md b/content/director-certs-openssl.md index 6269014f0..33d691761 100644 --- a/content/director-certs-openssl.md +++ b/content/director-certs-openssl.md @@ -1,3 +1,5 @@ +# Director SSL Certificate Configuration with OpenSSL + Depending on you configuration, there are up to three endpoints to be secured using SSL certificates: The Director, the UAA, and the SAML Service Provider on the UAA. !!! note @@ -65,6 +67,7 @@ ls -la . ``` --- + ## Configure the Director to use certificates {: #configure } Update the Director deployment manifest: @@ -113,6 +116,7 @@ If you are using the UAA for user management, additionally put certificates in t - Associated certificate for the UAA (e.g. content of `certs/uaa-sp.crt`) --- + ## Target the Director {: #target } After you deployed your Director with the above changes, you need to specify `--ca-cert` when targeting the Director: diff --git a/content/director-certs.md b/content/director-certs.md index af57ee413..6b86a1601 100644 --- a/content/director-certs.md +++ b/content/director-certs.md @@ -1,3 +1,5 @@ +# Configuring SSL Certificates + !!! note See [Director SSL Certificate Configuration with OpenSSL](director-certs-openssl.md) if you prefer to generate certs with OpenSSL config. @@ -49,7 +51,6 @@ cat certs.yml !!! note `duration` is set in days, and will default to 365 days - ## Configure the Director to use certificates {: #configure } Update the Director deployment manifest: @@ -98,6 +99,7 @@ If you are using the UAA for user management, additionally put certificates in t - Associated certificate for the UAA (content of `bosh int certs.yml --path /uaa_service_provider_ssl/certificate`) --- + ## Target the Director {: #target } After you deployed your Director with the above changes, you need to specify `--ca-cert` when targeting the Director: diff --git a/content/director-configure-blobstore.md b/content/director-configure-blobstore.md index 41aab45d6..06be6c6f5 100644 --- a/content/director-configure-blobstore.md +++ b/content/director-configure-blobstore.md @@ -1,9 +1,9 @@ +# Director Blobstore Configuration + The Director stores uploaded releases, configuration files, logs and other data in a blobstore. A default DAV blobstore is sufficient for most BOSH environments; however, a highly-available external blobstore may be desired. - - ## Included DAV (default) {: #included } By default the Director is configured to use included DAV blobstore job (see [Installing BOSH section](index.md#install) for example manifests). Here is how to configure it: @@ -42,6 +42,7 @@ By default the Director is configured to use included DAV blobstore job (see [In Above configuration is used by the Director and the Agents. --- + ## S3 {: #default } The Director and the Agents can use an S3 compatible blobstore. Here is how to configure it: @@ -74,6 +75,7 @@ The Director and the Agents can use an S3 compatible blobstore. Here is how to c ``` --- + ## Google Cloud Storage (GCS) {: #gcs } !!! note @@ -140,8 +142,9 @@ to store blobstore contents instead of the bucket default, specify `storage_clas json_key: | AGENT-SERVICE-ACCOUNT-BLOBSTORE-FILE ``` - + --- + ## Azure Storage Account {: #azure-storage } Azure Storage Account is supported from bosh version `278.0.0` and stemcell version `1.199`. diff --git a/content/director-configure-db.md b/content/director-configure-db.md index 8fcc135c3..cf6efc1b0 100644 --- a/content/director-configure-db.md +++ b/content/director-configure-db.md @@ -1,3 +1,5 @@ +# Director Database Configuration + The Director stores VM, persistent disk and other information in a database. An internal database might be sufficient for your deployment; however, a highly-available external database can improve performance, scalability and protect against data loss. ## Included Postgres (default) {: #included } @@ -35,6 +37,7 @@ The Director stores VM, persistent disk and other information in a database. An ``` --- + ## External {: #external } The Director is tested to be compatible with MySQL and Postgresql databases. diff --git a/content/director-tasks.md b/content/director-tasks.md index 9e8251121..e1a2ca2d4 100644 --- a/content/director-tasks.md +++ b/content/director-tasks.md @@ -1,3 +1,5 @@ +# Reviewing Tasks + An operator uses the CLI to interact with the Director. Certain CLI commands result in complex and potentially long running operations against the IaaS, blobstore, or other resources. Such commands are associated with a Director task and continue running on the Director even if the CLI disconnects from the Director. To find out if a CLI command has an associated Director task, look for "Director task [NUM]" in its output: @@ -8,7 +10,7 @@ bosh -d zookeeper deploy zookeeper.yml Should result in: -```text +```shell Using deployment 'zookeeper' Task 766 # <--- @@ -16,6 +18,7 @@ Task 766 # <--- ``` --- + ## Currently active tasks {: #active } At any time the Director might be performing multiple tasks at once. Active tasks can be in two states: `queued` or `processing`. @@ -28,15 +31,10 @@ bosh tasks Should result in: -``` -+-----+------------+-------------------------+-------+-------------------------------+--------+ | # | State | Timestamp | User | Description | Result | -+-----+------------+-------------------------+-------+-------------------------------+--------+ +|-----|------------|-------------------------|-------|-------------------------------|--------| | 766 | processing | 2015-01-27 21:39:30 UTC | admin | create deployment | | | 765 | queued | 2015-01-27 21:35:02 UTC | admin | scheduled SnapshotDeployments | | -+-----+------------+-------------------------+-------+-------------------------------+--------+ -``` - ### Joining tasks {: #join-active } @@ -48,7 +46,7 @@ bosh task 766 Should result in: -```text +```shell Director task 766 ...snip... ``` @@ -65,7 +63,7 @@ bosh task 766 --debug Should result in: -```text +```shell Director task 766 I, [2015-01-27T21:33:19.469158 #1769] [0x3fab30147330] INFO -- TaskHelper: Director Version: 1.2811.0 @@ -85,6 +83,7 @@ bosh cancel-task 766 ``` --- + ## Finished tasks {: #finished } The Director keeps a record of tasks that have finished. Finished tasks can be in two states: `done` or `error`. @@ -97,7 +96,7 @@ bosh tasks --recent Should result in: -``` +```text +-----+-------+-------------------------+--------+--------------------------+-----------------------------------------------------------+ | # | State | Timestamp | User | Description | Result | +-----+-------+-------------------------+--------+--------------------------+-----------------------------------------------------------+ @@ -115,7 +114,7 @@ Showing 30 recent tasks You can also run `bosh tasks --recent=NUM` to retrieve more tasks. !!! note - --all flag shows all tasks. Without that flag, the Director returns a subset of finished tasks that it deems important. + `--all` flag shows all tasks. Without that flag, the Director returns a subset of finished tasks that it deems important. ### Joining finished tasks {: #join-finished } diff --git a/content/director-users-uaa-scopes.md b/content/director-users-uaa-scopes.md index 86ab36666..26a5dafe5 100644 --- a/content/director-users-uaa-scopes.md +++ b/content/director-users-uaa-scopes.md @@ -1,3 +1,5 @@ +# Managing Permissions with UAA Scopes + If UAA is used for identity management, UAA will automatically verify a user's permissions when they log into the BOSH Director. @@ -10,6 +12,7 @@ as described in the access token. You can read more about UAA Access Tokens If you use the same private key to sign keys on different UAAs, users might obtain a token from one UAA and use it on the Director configured with a different UAA. It is therefore highly recommended to restrict scopes to individual Directors and not re-use a private key used for authenticating into UAA. --- + ## Logging into the Director as a User {: #user-login } Depending on how the UAA is configured different prompts may be shown. @@ -21,6 +24,7 @@ bosh login ``` --- + ## Logging into the Director as a UAA client {: #client-login } Non-interactive login, e.g. for scripts during a CI build is supported by the UAA by using a different UAA client allowing `client_credentials` grant type. @@ -34,6 +38,7 @@ bosh status See [the resurrector UAA client configuration](resurrector.md#uaa-client) for an example to set up an additional client. --- + ## Adding and Removing Users {: #uaac } The `uaa_admin` user, or another user with `clients.admin` or @@ -84,6 +89,7 @@ uaac member delete bosh.read some-new-user Changing group membership will take effect when a new access token is created for that user. New access are granted when their existing access token expires or when user logs out and logs in again. It is recommended to set access token validity to a short interval such as one minute. --- + ## Top-level Scopes {: #top-level-scopes } UAA scopes can be assigned to groups, individual users, or clients. @@ -103,6 +109,7 @@ uaac client get ``` --- + ### Full Admin {: #full-admin } Scopes: @@ -114,6 +121,7 @@ Scopes: Can use all commands on all deployments. --- + ### Full Read-only {: #full-read } Scopes: @@ -133,6 +141,7 @@ Can access in read-only capacity: - `bosh tasks`: list of all tasks summaries which includes task descriptions without access to debug logs --- + ### Stemcell uploader {: #stemcell-uploader } !!! note @@ -145,6 +154,7 @@ Scopes: Note that CLI will try to list stemcells before uploading given stemcell, hence `bosh upload stemcell` CLI command requires users/clients to have `bosh.read` scope as well. --- + ### Release uploader {: #release-uploader } !!! note @@ -157,6 +167,7 @@ Scopes: Note that CLI will try to list releases before uploading given release, hence `bosh upload release` CLI command requires users/clients to have `bosh.read` scope as well. --- + ### Anonymous {: #anon } Users with no UAA scopes are considered anonymous. @@ -166,9 +177,10 @@ Can access: - `bosh status`: show general information about targeted Director (authentication is not required) --- + ### Errors {: #errors } -``` +```text HTTP 401: Not authorized: '/deployments' requires one of the scopes: bosh.admin, bosh.UUID.admin, bosh.read, bosh.UUID.read ``` diff --git a/content/director-users-uaa.md b/content/director-users-uaa.md index 7577aa517..e441a0463 100644 --- a/content/director-users-uaa.md +++ b/content/director-users-uaa.md @@ -1,9 +1,12 @@ +# Configuring Director + !!! note This feature is available with bosh-release v209+ (1.3088.0) colocated with uaa v1+. In this configuration the Director is configured to delegate user management to the [UAA](https://github.com/cloudfoundry/uaa) server. The UAA server can be configured to manage its own list of users or work with an LDAP server, or a SAML provider. Regardless how the UAA server is configured the BOSH CLI will ask appropriate credentials and forward them to the UAA to request a token. --- + ## Deploy the Director with UAA {: #configure } 1. Change deployment manifest for the Director and add UAA release: diff --git a/content/director-users.md b/content/director-users.md index 429a980a6..40145110d 100644 --- a/content/director-users.md +++ b/content/director-users.md @@ -1,6 +1,9 @@ +# Using Basic Users + The Director provides a very simple built-in user management system for authentication of operators and internal services (for example, the Health Monitor). Alternatively, it can integrate with UAA for more advanced use cases. --- + ## Default Configuration {: #default } !!! note @@ -32,6 +35,7 @@ bosh delete user some-operator ``` --- + ## Preconfigured Users {: #preconfigured } !!! note @@ -57,16 +61,19 @@ To configure the Director with a list of users: 1. Redeploy the Director with the updated manifest. --- + ## UAA Integration {: #uaa } [Configure the Director with UAA user management](director-users-uaa.md). --- + ## Director Tasks {: #hm } When a user initiates a [director task](director-tasks.md), the director logs the user in the task audit log. --- + ## Health Monitor Authentication {: #hm } The Health Monitor is configured to use a custom user to query/submit requests to the Director. Since by default the Director does not come with any users, the Health Monitor is not able to successfully communicate with the Director. See the [Automatic repair with Resurrector](resurrector.md) topic for more details. diff --git a/content/disaster-recovery.md b/content/disaster-recovery.md index 538c4b995..8711e6ddd 100644 --- a/content/disaster-recovery.md +++ b/content/disaster-recovery.md @@ -5,13 +5,18 @@ This has not always been necessary since BOSH is a deploytime-relevant service, However, it is possible that we externalize the state to the best possible extent to highly available infrastructure offerings, and fallback to another AZ in case of an AZ outage. This can be achieved if certain prerequisites are fulfilled. -## Prerequisites: +## Prerequisites ### 1. Using domain names + When an AZ outage happens, and we switch to a new director, it is important that BOSH's deployments are still able to reach the new director in the other AZ. The first step to make this possible is to use domain names for the director instead of IPs. This, however, does come with a challenge of updating the the certificates used by BOSH with alternative names (IPs being used to connect to the director and new hostname configured) for hostname validation. + ### 2. Externalizing the database + BOSH should still be aware of its deployments, their instances, blob references etc. in the event of its redeployment to another AZ. This could be made possible by using an external database instead of the default colocated configuration. This can be done by consuming highly available RDS services by IaaS providers, and then configuring the director database properties and the UAA database properties to consume these external database instances. See [director db configuration](https://bosh.io/docs/director-configure-db) and [uaa configuration](https://bosh.io/docs/director-users-uaa) for further help in this regard. + ### 3. Externalizing the blobstore + It would also make sense to use an external blobstore like AWS S3 which is highly available than consuming blobs from the colocated DAV blobstore. This would help to fail over to a new director with less worries of re-fetching the blobs (keeping in mind this is a time consuming process) when the need arises. As of now, it is already possible to use highly available blobstores provided by AWS, Azure, AliCloud and GCP. Switching over to a new blobstore can be smooth if there's a blobstore migration mechanism implemented using the existing BOSH Blobstore CLIs (by simply 'putting' the blobs using the s3cli from the director VM to the S3 bucket for example in case of AWS). Another way of doing this would be to recreate the blobstore externally by uploading the blobs with [--fix](https://bosh.io/docs/cli-v2/#upload-release) or running the deploy commands with [--fix-releases](https://bosh.io/docs/cli-v2/#deploy). This however is costly in terms of time required to re-fetch the blobs. See [this](https://bosh.io/docs/director-configure-blobstore) to start configuring the director to use an external blobstore. @@ -20,10 +25,13 @@ See [this](https://bosh.io/docs/director-configure-blobstore) to start configuri We can assume that we are working with multiple deployments across two availability zones: `zone_1` and `zone_2`, and that our director is currently deployed in `zone_1`. Then, let's also assume that there's a sudden outage in `zone_1`. ### 1. Isolating the director + To isolate the director in `zone_1` from the database to prevent it from modifying the state when the zone is up, it is important that:\ a. The database passwords are rotated\ b. All existing database connections are killed via psql (by executing [pg_terminate_backend](https://www.postgresql.org/docs/current/functions-admin.html) for example) + ### 2. Deploying another director + There might be differences in deploying BOSH directors in another AZ based on whether we use the `create-env` approach or the `bosh deploy` approach. After isolating the director deployed by `create-env` in `zone_1` the process to deploy a second one in `zone_2` can be treated similar to a scratch installation, implying that the zone used in the director's deployment manifest is changed to `zone_2` and the `bosh-state.json` is dropped before calling [create-env](https://bosh.io/docs/cli-v2/#create-env) and setting up the second director in `zone_2`. @@ -34,12 +42,12 @@ However, while redeploying the director with the updated manifest, the director first try to delete the old `bosh` in `zone_1`. This will most probably fail in case of a zone outage. This can be prevented by invoking [ignore](https://bosh.io/docs/cli-v2/#ignore) on the deployed `bosh` in `zone_1`. ### 3. Disabling resurrection + After `bosh` is successfully deployed in `zone_2` it will now take care for repairing its deployments. Most of them would be in error situation since they would have instances in the failing zone. The director would continuously trigger `scan_and_fix` -tasks which would then also fail. +tasks which would then also fail. Therefore, an option would be to deploy the new directors with disabled resurrection. After the zone outage is over, the directors in `zone_1` can be dropped and the one in `zone_2` can be used further. Of course with enabled resurrection. [Resurrection](https://bosh.io/docs/cli-v2/#update-resurrection) can be enabled/disabled via `bosh update-resurrection on (or off)`. - [>Here<](https://www.youtube.com/watch?v=0oMrGu9XuBY&list=PLhuMOCWn4P9jUHBucZBkSjmkwEbvx8vxf&index=12) is a video presentation on the topic from the 2023 CF Day at Heidelberg. diff --git a/content/dns.md b/content/dns.md index ee21f361e..9e49f3114 100644 --- a/content/dns.md +++ b/content/dns.md @@ -1,3 +1,5 @@ +# Native DNS Support + Using DNS within deployments: - allows easy use of dynamic networks since IPs change with every redeploy @@ -8,25 +10,27 @@ Using DNS within deployments: See [links](links.md) for more context on how to use links with BOSH. --- + ## Architecture {: #arch } To provide native DNS support: - Director keeps track of DNS entries assigned to each instance (within the `.bosh` default TLD or a custom TLD set in [dns.domain_name](https://bosh.io/jobs/director?source=github.com/cloudfoundry/bosh#p%3ddns.domain_name)) - Agent updates DNS records metadata on its VM, as well as `/etc/hosts` for Ubuntu and `C:\windows\System32\drivers\etc\hosts` for Windows -- DNS release (more details below) provides resolution of BOSH specific DNS records by exposing a simple DNS server +- DNS release (more details below) provides resolution of BOSH specific DNS records by exposing a simple DNS server Given that the Director is the sole orchestrator of the system, it is now responsible for updating DNS records during a deploy. As VMs are created and deleted following DNS related steps happen: 1. Director notices that VM, after it's created or deleted, changed its IP -1. Director creates a new DNS records dataset and saves it to the blobstore +1. Director creates a new DNS records dataset and saves it to the blobstore 1. Director issues sync_dns Agent call to *all* VMs (in all deployments) -1. Each Agent downloads new DNS records dataset and updates `/var/vcap/instance/dns/records.json` +1. Each Agent downloads new DNS records dataset and updates `/var/vcap/instance/dns/records.json` 1. DNS release sees that local `/var/vcap/instance/dns/records.json` is updated, hence returns new information in future DNS requests served, according to the [supported query syntax](dns.md#constructing-queries) See [Deploying step-by-step](deploying-step-by-step.md) for full Director deployment flow. --- + ## Types of DNS addresses {: #dns-addresses } There are two types of DNS addresses that native DNS supports: @@ -37,6 +41,7 @@ There are two types of DNS addresses that native DNS supports: Since BOSH DNS is automatically managed, DNS addresses are not meant to be constructed manually by operators or scripts. To obtain a DNS address you can use Links API or job template accessors within your jobs. --- + ## DNS release {: #dns-release } To take advantage of native DNS functionality, it's expected that [DNS release](https://bosh.io/releases/github.com/cloudfoundry/bosh-dns-release?all=1) runs on each VM. We recommend to colocate DNS release by defining it in an [addon](runtime-config.md#addons). @@ -52,17 +57,18 @@ On Ubuntu stemcells prior to Noble, BOSH DNS is designed to be the primary DNS s Here is how DNS release chooses recursors before starting its operation: 1. by default will pick up recursors specified in `/etc/resolv.conf` (denoted by `nameserver` keyword) - - alternatively, if `recursors` property is set use specified recursors + - alternatively, if `recursors` property is set use specified recursors 1. exclude recursors specified in `excluded_recursors` property 1. pick a recursor from the list of recursors 1. if you have a version of the DNS release after and including 1.12, the selection of recursors is based on `recursor_selection` strategy: - - if `recursor_selection` is "smart": - - note that all recursors in this list will be considered equivalent, i.e. able to resolve same domains - - if `recursor_selection` is "serial": - - the next recursor (in order) from the list of recursors is chosen + - if `recursor_selection` is "smart": + - note that all recursors in this list will be considered equivalent, i.e. able to resolve same domains + - if `recursor_selection` is "serial": + - the next recursor (in order) from the list of recursors is chosen + 1. if you have a version before 1.12, then the behaviour is the same as having `recursor_selection` set to "smart" from above 1. failover to using another recursor, if current recursor fails - - if you have a version before 1.18 connectivity problems do not account for resolution problems (NXDOMAIN, or other DNS level errors) + - if you have a version before 1.18 connectivity problems do not account for resolution problems (NXDOMAIN, or other DNS level errors) #### More on recursor_selection @@ -98,15 +104,15 @@ DNS release allows operators to specify custom names for BOSH generated DNS reco There are two ways to configure aliases: -0. Installing a `dns/aliases.json` file through your own release job (or through your BOSH manifest using the [bosh-dns-aliases-release](https://github.com/cloudfoundry/bosh-dns-aliases-release)). By default, `bosh-dns` will glob local `/var/vcap/jobs/*/dns/aliases.json` files for aliases. -0. Statically configuring the [`aliases` property](https://bosh.io/jobs/bosh-dns?source=github.com/cloudfoundry/bosh-dns-release#p=aliases) of the `bosh-dns` job running the DNS server. +1. Installing a `dns/aliases.json` file through your own release job (or through your BOSH manifest using the [bosh-dns-aliases-release](https://github.com/cloudfoundry/bosh-dns-aliases-release)). By default, `bosh-dns` will glob local `/var/vcap/jobs/*/dns/aliases.json` files for aliases. +1. Statically configuring the [`aliases` property](https://bosh.io/jobs/bosh-dns?source=github.com/cloudfoundry/bosh-dns-release#p=aliases) of the `bosh-dns` job running the DNS server. The alias configuration should be a hash, with keys representing the alias and array values representing the target hostnames. Target hostnames will be resolved and merged before sending the results back to the client. There are two special characters which can be used (see below for example usages of them): - * asterisk (`*`) - used in target hostnames to match subdomains - * underscore (`_`) - represents a subdomain and can be used to match the subdomain in the target hostname (useful for queries needing to resolve instance IDs) +- asterisk (`*`) - used in target hostnames to match subdomains +- underscore (`_`) - represents a subdomain and can be used to match the subdomain in the target hostname (useful for queries needing to resolve instance IDs) #### Example @@ -123,9 +129,9 @@ Using the following aliases configuration: The following queries demonstrate the expected resolution behaviors: - * `sql-db.service.cf.internal` will internally resolve to all VMs in the `mysql-z1` and `mysql-z2` instance groups. - * `myuuid.cell.service.cf.internal` might internally resolve to `myuuid.windows-cell.default.cf.bosh`, assuming `windows-cell` has an instance with a UUID of `myuuid`. - * `_.cell.service.cf.internal` (literal query) will not resolve since it is different than asterisk aliases. +- `sql-db.service.cf.internal` will internally resolve to all VMs in the `mysql-z1` and `mysql-z2` instance groups. +- `myuuid.cell.service.cf.internal` might internally resolve to `myuuid.windows-cell.default.cf.bosh`, assuming `windows-cell` has an instance with a UUID of `myuuid`. +- `_.cell.service.cf.internal` (literal query) will not resolve since it is different than asterisk aliases. !!! tip Aliases are very useful when [migrating from Consul](dns.md#migrate-consul). @@ -184,7 +190,7 @@ instance_groups: ###### Basic alias -A basic alias is an _unparameterized_ alias on a _constant domain_ with a _constant_ query. It returns all IPs matching the filter that provide that link. +A basic alias is an *unparameterized* alias on a *constant domain* with a *constant* query. It returns all IPs matching the filter that provide that link. Example: @@ -196,7 +202,8 @@ aliases: ``` ###### Wildcard alias -A wildcard alias is an _unparameterized_ alias on a _wildcard domain_ with a _constant_ query. + +A wildcard alias is an *unparameterized* alias on a *wildcard domain* with a *constant* query. It returns all IPs matching the filter that provide that link. Example: @@ -209,8 +216,9 @@ aliases: ``` ###### Placeholder alias -A placeholder alias is a _parameterizable_ alias on a _wildcard domain_ with a -_variable_ query. It returns IPs matching both the filter that provides that + +A placeholder alias is a *parameterizable* alias on a *wildcard domain* with a +*variable* query. It returns IPs matching both the filter that provides that link and the placeholder replacement. It allows referencing a placeholder (_) specified in the alias. The type of @@ -240,7 +248,7 @@ aliases: ###### Parameters in Detail -**domain** [String] (*required*) +**domain** [String] (*required*) Describes the domain name the alias should return results for when queried. @@ -267,19 +275,18 @@ Examples: If present, filters the results to only return jobs matching the specified health status, e.g. only healthy ones, unhealthy ones, or all of them. -- `smart` (default) returns only healthy or unchecked jobs; however, if all the jobs in an instance_group are unhealthy all of them are returned. +- `smart` (default) returns only healthy or unchecked jobs; however, if all the jobs in an instance_group are unhealthy all of them are returned. -- `healthy` returns only healthy jobs +- `healthy` returns only healthy jobs -- `unhealthy` returns only unhealthy - -- `all` returns all jobs regardless of their health +- `unhealthy` returns only unhealthy +- `all` returns all jobs regardless of their health **initial_health_check** [String] (*optional*) -Because BOSH has to start tracking a given job's health status, by default it will return all (unfiltered) IPs on the very first request and asynchronously begin tracking their health. -Setting this to `synchronous` will force BOSH to wait for the first health statuses to come in and filter by them. This will take longer, but guarantees that health has been checked at least once even for the very first DNS request. +Because BOSH has to start tracking a given job's health status, by default it will return all (unfiltered) IPs on the very first request and asynchronously begin tracking their health. +Setting this to `synchronous` will force BOSH to wait for the first health statuses to come in and filter by them. This will take longer, but guarantees that health has been checked at least once even for the very first DNS request. - `asynchronous` (default) will return unchecked results to smart queries and begin health-checking those IP addresses in the background @@ -314,6 +321,7 @@ instance_groups: ``` Both proxied and direct define api.bosh.internal as an alias, but with different filters. Resolving the api.bosh.internal address would return both: + 1. IPs of VMs for the proxied instance group in which the nginx job is healthy or unchecked 1. IPs of VMs for the direct instance group in which the web job is healthy (checking the health synchronously on the first request) @@ -321,7 +329,7 @@ Both proxied and direct define api.bosh.internal as an alias, but with different DNS release provides a way to enable response caching based on response TTLs. Enabling caching typically will alleviate some pressure from your upstream DNS servers and decrease latency of DNS resolutions. -To enable caching, use `cache.enabled` property. Canonical DNS runtime config with caching enabled can be found here: https://github.com/cloudfoundry/bosh-deployment/blob/master/runtime-configs/dns.yml. +To enable caching, use `cache.enabled` property. Canonical DNS runtime config with caching enabled can be found in the [dns.yml](https://github.com/cloudfoundry/bosh-deployment/blob/master/runtime-configs/dns.yml). ### Additional Handlers @@ -340,7 +348,9 @@ logging: - name: ForwardHandler log_level: DEBUG ``` + --- + ## Enabling DNS {: #enable } To enable native BOSH functionality, you must first enable [`local_dns.enabled` property](https://bosh.io/jobs/director?source=github.com/cloudfoundry/bosh#p=director.local_dns.enabled) in the Director job. See [bosh-deployment's bosh.yml](https://github.com/cloudfoundry/bosh-deployment/blob/90bac489919fd4512bc9bb4d24070d71b07cd586/bosh.yml#L92-L93) as an example. @@ -352,16 +362,21 @@ If you were relying on instance index based DNS records, you must enable [`local Additionally, you should colocate DNS release via an addon in all your deployments. See [bosh-deployment's runtime-configs/dns.yml](https://github.com/cloudfoundry/bosh-deployment/blob/master/runtime-configs/dns.yml) as an example. --- + ## Disabling DNS {: #disable } + To disable the native BOSH functionality, you must disable the [`local_dns.enabled` property](https://bosh.io/jobs/director?source=github.com/cloudfoundry/bosh#p=director.local_dns.enabled) in the Director job and remove the addon for the DNS release. **Note:** Because of a known issue in [the `bosh-dnsrelease`](https://github.com/cloudfoundry/bosh-dns-release/issues/34) you have to recreate all VMs afterwards, in order to remove the local DNS server from `/etc/resolv.conf`. --- + ## DNS Monitoring {: #monitoring } + DNS release provides monitoring, which can be enabled with [`metrics.enabled` property](https://bosh.io/jobs/bosh-dns?source=github.com/cloudfoundry/bosh-dns-release#p%3dmetrics.enabled) or with this [addon config](https://github.com/cloudfoundry/bosh-deployment/blob/master/misc/dns-addon-enable-local-monitoring.yml). By default the metrics endpoint will be exposed on `http://127.0.0.1:53088/metrics`. The bind address for the metrics server could be change with [`metrics.address` property](https://bosh.io/jobs/bosh-dns?source=github.com/cloudfoundry/bosh-dns-release#p%3dmetrics.address) and the port with [`metrics.port` property](https://bosh.io/jobs/bosh-dns?source=github.com/cloudfoundry/bosh-dns-release#p%3dmetrics.port) or with this [addon config](https://github.com/cloudfoundry/bosh-deployment/blob/master/misc/dns-addon-enable-external-monitoring.yml). --- + ## Impact on links {: #links } Each link includes some networking information about its provider. Addresses returned by a link may be either IP addresses or DNS addresses. @@ -408,11 +423,13 @@ link("db").instances[0].address => "ef489dd9-48f6-45f0-b7af-7f3437919b17.db.defa ``` --- + ## Impact on job's address (`spec.address`) {: #job-address } Similar to how [links are affected](dns.md#links), `spec.address` will start returning DNS address once `use_dns_addresses` feature is enabled. --- + ## Migrating from PowerDNS {: #migrate-powerdns } Historically BOSH users did not have an easy highly available solution to enable DNS for their deployments. PowerDNS was a possible choice; however, it required more advanced configuration that we felt comfortable recommending to everyone. We are planning to deprecate and remove PowerDNS integration. To migrate from PowerDNS to native DNS: @@ -423,6 +440,7 @@ Historically BOSH users did not have an easy highly available solution to enable 1. redeploy Director without `powerdns` job --- + ## Migrating from Consul {: #migrate-consul } To ease migration from Consul DNS entries, DNS release provides [aliases feature](dns.md#aliases). It allows operators to define custom DNS entries that can map to BOSH generated DNS entries. To migrate off of Consul to native DNS: @@ -434,6 +452,7 @@ To ease migration from Consul DNS entries, DNS release provides [aliases feature 1. redeploy all deployments without `consul_agent` job --- + ## Constructing DNS Queries {: #constructing-queries } BOSH DNS provides its own structured query language for querying instance IP addresses @@ -441,23 +460,24 @@ based on an instance's endemic and organizational relationship; e.g., by an instance's healthiness, its availability zone, or group id. Supported DNS records follow the format `....` with: -* ``: a query part starting with `q-` followed by a number of parameters detailed below (e.g. health, az or instance/network uid) -* ``: the name of the instance group to include or `*` to include all instance groups -* ``: the name of the network to include or `*` to include IP addresses from all networks -* ``: the name of the deployment to include or `*` to include all deployments -* `` the top-level-domain configured within [dns.domain_name](https://bosh.io/jobs/director?source=github.com/cloudfoundry/bosh#p%3ddns.domain_name) which defaults to `.bosh` + +- ``: a query part starting with `q-` followed by a number of parameters detailed below (e.g. health, az or instance/network uid) +- ``: the name of the instance group to include or `*` to include all instance groups +- ``: the name of the network to include or `*` to include IP addresses from all networks +- ``: the name of the deployment to include or `*` to include all deployments +- `` the top-level-domain configured within [dns.domain_name](https://bosh.io/jobs/director?source=github.com/cloudfoundry/bosh#p%3ddns.domain_name) which defaults to `.bosh` A few example DNS queries: -* `dig @bosh-dns q-s3.zookeeper.*.zk-prod.bosh` +- `dig @bosh-dns q-s3.zookeeper.*.zk-prod.bosh` Query for BOSH instances that are healthy (`q-s3`), from instance group `zookeeper`, on all networks (`*`), in deployment `zk-prod`. -* `dig @bosh-dns q-s1-a2.diego-cell.*.*.bosh` +- `dig @bosh-dns q-s1-a2.diego-cell.*.*.bosh` Query for BOSH instances that are unhealthy (`q-s1`), that are in availability zone 2 (`q-a2`), from instance group `diego_cell`, on any network (`*`) and any deployment (`*`). **Note:** BOSH DNS converts instance groups with underscores in their name to hyphens (e.g. `diego_cell` becomes `diego-cell`). -* `dig @bosh-dns q-s4.*.bosh` +- `dig @bosh-dns q-s4.*.bosh` Query for all BOSH instances regardless of healthiness (`q-s4`). This effectively returns all instances across all deployments on the BOSH director. @@ -469,31 +489,28 @@ dig @bosh-dns q-a*i*m*n*s*y*.q-g*.your-domain.bosh. Query parameters are: -* `a*` = availability zone - * where `*` is the numerical id of the availability zone -* `i*` = instance id -* `m*` = numerical uuid -* `n*` = network - * where `*` is the numerical id of the network -* `s*` = healthiness - * The following options are available: - * `s0` - _Default_ - 'smart' strategy that returns healthy and unchecked instances; if - there are no healthy or unchecked instances, all instances will be returned - * `s1` - returns only unhealthy instances - * `s3` - return only healthy instances - * `s4` - return all instances -* `y*` = synchronous healthcheck - * The following options are available: - * `y0` - _Default_ - do not attempt to get healthiness on the first query - * `y1` - Perform a synchronous health check the first time the record is - resolved. This is useful for applications that are not designed to continuously - re-resolve and therefore need to receive a healthy instance on the first - record resolution. -* `g*` = group (internal) - * where `*` is the global instance group id - * this flag is used almost exclusively for debugging purposes only +- `a*` = availability zone + - where `*` is the numerical id of the availability zone +- `i*` = instance id +- `m*` = numerical uuid +- `n*` = network + - where `*` is the numerical id of the network +- `s*` = healthiness + - The following options are available: + - `s0` - *Default* - 'smart' strategy that returns healthy and unchecked instances; if there are no healthy or unchecked instances, all instances will be returned + - `s1` - returns only unhealthy instances + - `s3` - return only healthy instances + - `s4` - return all instances +- `y*` = synchronous healthcheck + - The following options are available: + - `y0` - *Default* - do not attempt to get healthiness on the first query + - `y1` - Perform a synchronous health check the first time the record is resolved. This is useful for applications that are not designed to continuously re-resolve and therefore need to receive a healthy instance on the first record resolution. +- `g*` = group (internal) + - where `*` is the global instance group id + - this flag is used almost exclusively for debugging purposes only --- + ## Consuming BOSH DNS in Job Templates {: #consuming-dns-job-templates } BOSH DNS' query language is not meant to be manually crafted. As a release @@ -516,25 +533,26 @@ the `db` group is of id `5`. The following options are available when constructing a link query: - * `azs` (`a*`): list of availability zone names - * `uuid` (`m*`): instance uuid - * `status` (`s*`): health status. Can be one of `default`, `healthy`, `unhealthy`, or `all` - * `default_network` (`n*`): network name - * `instance_group` (`g*`): instance group name - * `deployment_name` (`g*`): deployment name +- `azs` (`a*`): list of availability zone names +- `uuid` (`m*`): instance uuid +- `status` (`s*`): health status. Can be one of `default`, `healthy`, `unhealthy`, or `all` +- `default_network` (`n*`): network name +- `instance_group` (`g*`): instance group name +- `deployment_name` (`g*`): deployment name --- + ## BOSH DNS Addresses in Config Server Generated Certs {: #dns-variables-integration} !!! note This feature is still in alpha phase. -With BOSH `v267+`, [Config Server](variable-types.md) generated certificates can be optionally created with automatic BOSH DNS records in their Common Name and/or Subject Alternative Names. +With BOSH `v267+`, [Config Server](variable-types.md) generated certificates can be optionally created with automatic BOSH DNS records in their Common Name and/or Subject Alternative Names. A [variable](variable-types.md) of type `certificate` can now **explicitly** consume two links: 1. **Name:** `alternative_name`, **Type:** `address`. When consumed, the BOSH DNS address of the link provider will be added to the Subject Alternative Names of the generated certificate. -1. **Name:** `common_name`, **Type:** `address`. When consumed, the BOSH DNS address of the link provider will be set as the Common Name of the generated certificate **ONLY IF** the variable definition does not specify a common name. If the variable definition specifies a common name, it will **NOT** be overridden. +1. **Name:** `common_name`, **Type:** `address`. When consumed, the BOSH DNS address of the link provider will be set as the Common Name of the generated certificate **ONLY IF** the variable definition does not specify a common name. If the variable definition specifies a common name, it will **NOT** be overridden. **Note that the above 2 links are optional.** @@ -611,19 +629,19 @@ variables: Which will result in the variable called `app_server_cert` having a SAN set to -* DNS: `*.server_ig.default.app-service.bosh`. +- DNS: `*.server_ig.default.app-service.bosh`. ### When Variable Definition has SANS and/or CN Defined in its Options -If the variable of type certificate defines a list of Subject alternative Names in its options, and at the same time it consumes the `alternative_name` link, the BOSH DNS address of the provider will be added to the list SANs in the generated certificate. +If the variable of type certificate defines a list of Subject alternative Names in its options, and at the same time it consumes the `alternative_name` link, the BOSH DNS address of the provider will be added to the list SANs in the generated certificate. -In contrast, if the variable of type certificate defines a Common Name in its options, and at the same time it consumes the `common_name` link, the BOSH DNS address of the provider will **NOT** override the Common Name defined in options. +In contrast, if the variable of type certificate defines a Common Name in its options, and at the same time it consumes the `common_name` link, the BOSH DNS address of the provider will **NOT** override the Common Name defined in options. For example, the `app_server_cert` cert below will have "**Application Server**" as Common Name, and will have the following SANs: - - * DNS: `custom-record.appservers.cf.local` - * DNS: `*.serverig.default.app-service.bosh` - * IP: 172.158.20.255 + +- DNS: `custom-record.appservers.cf.local` +- DNS: `*.serverig.default.app-service.bosh` +- IP: 172.158.20.255 ```yaml variables: @@ -642,13 +660,15 @@ variables: !!! Warning In order for the variables to be regenerated by Config Server(usually Credhub) when any of their options changes, the [`update_mode`](manifest-v2.md#variables) option should be set to `converge` in the deployment manifest. - + --- + ## Rotating BOSH DNS Certificates {: #rotating-dns-certificates } BOSH DNS Health Monitor Certificates should be performed in three steps in order to achieve zero downtime. Given you used bosh-deployment to update your runtime config as in: + ```shell bosh update-runtime-config bosh-deployment/runtime-configs/dns.yml --vars-store bosh-dns-certs.yml ``` @@ -779,6 +799,7 @@ bosh update-runtime-config bosh-deployment/runtime-configs/dns.yml --vars-store Redeploy all VMs. -- + ## Instance `records.json` Data Each VM receives a local copy of the latest DNS data (via the BOSH agent on the VM) whenever VMs are added or removed from the system. This data file is installed to `/var/vcap/instance/dns/records.json`. Below is an example of the schema... diff --git a/content/drain.md b/content/drain.md index 52090e176..ebb44e7b0 100644 --- a/content/drain.md +++ b/content/drain.md @@ -1,10 +1,14 @@ +# Drain + (See [Job Lifecycle](job-lifecycle.md) for an explanation of when drain scripts run.) Release job can have a drain script that will run when the job is restarted or stopped. This script allows the job to clean up and get into a state where it can be safely stopped. For example: + - when writing a release for a load balancer, each node can safely stop accepting new connections and drain existing connections before fully stopping. - when writing a release for a database or other distributed datastore, you can ensure that data is correctly replicated/distributed on all nodes before stopping the node. --- + ## Job Configuration {: #job-configuration } To add a drain script to a release job: @@ -25,6 +29,7 @@ templates: Drain script from each release job will run if they are deployed on 3093+ stemcells. Before only the first release job's drain script ran. --- + ## Script Implementation {: #script-implementation } Drain script is usually just a regular shell script. Since drain script is executed in a similar way as other release job scripts (start, stop, pre-start scripts) you can use job's package dependencies. @@ -55,12 +60,13 @@ You must ensure that your drain script exits in one of following ways: Note that if drain script causes monitored job processes to exit, monit will not call stop script for that job. --- + ## Environment Variables {: #environment-variables } Drain script can access the following environment variables: -* `BOSH_JOB_STATE`: JSON description of the current job state -* `BOSH_JOB_NEXT_STATE`: JSON description of the new job state that is being applied +- `BOSH_JOB_STATE`: JSON description of the current job state +- `BOSH_JOB_NEXT_STATE`: JSON description of the new job state that is being applied Currently, only persistent disk size is provided in those two variables. For example: @@ -98,27 +104,33 @@ You'll find [here](https://github.com/cloudfoundry-incubator/cfcr-etcd-release/b an example script for an etcd member to leave its etcd cluster gracefully. --- + ## Command-line arguments {: #command-line-arguments } The first argument passed to the drain script indicates the intended job lifecycle action. It can have the following values (note: "job" here actually means "instance"): -* `job_changed` indicating that the instance will be restarted -* `job_shutdown` indicating that the instance will be stopped and subsequently the VM will terminated + +- `job_changed` indicating that the instance will be restarted +- `job_shutdown` indicating that the instance will be stopped and subsequently the VM will terminated The second argument passed to the drain script indicates whether the job hash has changed. It can have the following values: -* `hash_changed` indicating that the job's packages or rendered templates have changed. -* `hash_unchanged` indicating that the job's packages and rendered templates have not changed. + +- `hash_changed` indicating that the job's packages or rendered templates have changed. +- `hash_unchanged` indicating that the job's packages and rendered templates have not changed. --- + ## Logs {: #logs } Currently logs from the drain script are not saved on disk by default, though release author may choose to do so explicitly. We are planning to eventually make it more consistent with [pre-start script logging](pre-start.md#logs). --- + ## Examples {: #example } ### Load-balancer + ```shell #!/bin/bash @@ -137,7 +149,8 @@ kill -USR2 $pid echo 15; exit 0 ``` -# Stateful distributed job +## Stateful distributed job + ```shell #!/bin/bash diff --git a/content/errands.md b/content/errands.md index ed343dc1f..15e288bdf 100644 --- a/content/errands.md +++ b/content/errands.md @@ -1,3 +1,5 @@ +# Running Errands + (See [Jobs](jobs.md) for an introduction to jobs.) Any job that includes `bin/run` script in its spec file's templates section is considered to be an errand. Operator can trigger execution of an errand at any time after the deploy and receive back script's stdout, stderr and exit code upon its completion. @@ -5,6 +7,7 @@ Any job that includes `bin/run` script in its spec file's templates section is c Errand output is limited to one megabyte of data. If the response is larger an error will be returned instead. Release authors should ensure the full error is properly logged to disk and a summary is returned as output. --- + ## Release Definition {: #release-definition } Example of an errand job `smoke-tests` from Zookeeper release. `bin/run` script is specified in its templates section: @@ -39,6 +42,7 @@ export ZOOKEEPER_SERVERS=<%= conn.instances.map { |i| "#{i.address}:#{conn.p('cl ``` --- + ## Include in a Deployment {: #include-in-deployment } There are two ways to add an errand to a deployment: @@ -86,6 +90,7 @@ Alternatively, it might make sense to colocate an errand job with other jobs in ``` --- + ## Execution {: #execution } Unlike regular jobs which run continuously and get automatically restarted on failure, errand jobs are executed upon operator's request some time after a deploy and if fail do not get restarted. There is no timeout on how long an errand can execute. @@ -100,7 +105,7 @@ bosh -e vbox -d zookeeper errands Should result in: -```text +```shell Using environment '192.168.56.6' as client 'admin' Using deployment 'zookeeper' @@ -122,7 +127,7 @@ bosh -e vbox -d zookeeper run-errand status Should result in: -```text +```shell Using environment '192.168.56.6' as client 'admin' Using deployment 'zookeeper' @@ -193,7 +198,7 @@ Succeeded If an errand job is colocated on multiple instances (over one or more instance groups), by default `bosh run-errand` command will execute them all in parallel. You can limit number of instances used for execution via `--instance` flag: -``` +```shell bosh -e vbox -d zookeeper run-errand status --instance zookeeper/3e977542-d53e-4630-bc40-72011f853cb5 ``` diff --git a/content/events.md b/content/events.md index 5dc8dcad3..69df43aeb 100644 --- a/content/events.md +++ b/content/events.md @@ -1,3 +1,5 @@ +# Auditing Events + !!! note This feature is available in bosh-release v256+. @@ -18,7 +20,7 @@ Run [`bosh events` command](sysadmin-commands.md#events) to view 200 recent even bosh events ``` -```text +```shell +--------------+------------------------------+-------+-------------+-------------+------------------------------------------------+------+-----------+------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ | ID | Time | User | Action | Object type | Object ID | Task | Dep | Inst | Context | +--------------+------------------------------+-------+-------------+-------------+------------------------------------------------+------+-----------+------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -74,6 +76,7 @@ bosh events --instance zookeeper/ca5f695a-eb81-49fd-a577-33825cb1b5fc ``` --- + ## Ending vs. Single Actions {: #ending-vs-single } Each event represents an action. Some actions take time to perform (e.g. delete a VM), and other actions are just one-off events (e.g. set up SSH access). Actions that take time are represented by two events (starting and ending one) instead of just one. In the example below **delete VM** action is recorded as starting in event #5096 and finishing in event #5199. @@ -84,6 +87,7 @@ Each event represents an action. Some actions take time to perform (e.g. delete ``` --- + ## Enabling Event Collection {: #enable } To enable this feature: diff --git a/content/flush-arp.md b/content/flush-arp.md index 930e1d794..6fb0493c0 100644 --- a/content/flush-arp.md +++ b/content/flush-arp.md @@ -1,3 +1,5 @@ +# Explicit ARP Flushing + !!! note This feature is available with bosh-release v256+ and 3232+ stemcell series. diff --git a/content/google-cpi.md b/content/google-cpi.md index 01a7bd328..9c5073abd 100644 --- a/content/google-cpi.md +++ b/content/google-cpi.md @@ -1,11 +1,13 @@ +# Google CPI Usage + This topic describes cloud properties for different resources created by the Google CPI. ## AZs {: #azs } Schema for `cloud_properties` section: -* **zone** [String, required]: [Zone](https://cloud.google.com/compute/docs/regions-zones/regions-zones) to use for creating VMs. Example: `us-central1-f`. -* **node_group** [String, optional]: Name of the [Sole Tenant Group](https://cloud.google.com/compute/docs/nodes/sole-tenant-nodes) in which VMs will be created. +- **zone** [String, required]: [Zone](https://cloud.google.com/compute/docs/regions-zones/regions-zones) to use for creating VMs. Example: `us-central1-f`. +- **node_group** [String, optional]: Name of the [Sole Tenant Group](https://cloud.google.com/compute/docs/nodes/sole-tenant-nodes) in which VMs will be created. Example: @@ -17,16 +19,17 @@ azs: ``` --- + ## Networks {: #networks } Schema for `cloud_properties` section used by dynamic network or manual network subnet: -* **network\_name** (String, optional) - The name of the [Google Compute Engine Network](https://cloud.google.com/compute/docs/networking#networks) the CPI will use when creating the instance (if not set, by default it will use the `default` network). Example: `cf`. -* **xpn\_host\_project\_id** (String, optional) - The [project id](https://support.google.com/cloud/answer/6158840?hl=en) that owns the network resource to support [Shared VPC Networks (XPN)](https://cloud.google.com/compute/docs/xpn/) (if not set, it will default to the project hosting the compute resources). Example: `my-other-project`. -* **subnetwork\_name** (String, optional) - The name of the [Google Compute Engine Subnet Network](https://cloud.google.com/compute/docs/networking#subnet_network) the CPI will use when creating the instance. If the network is in legacy mode, do not provide this property. If the network is in auto subnet mode, providing the subnetwork is optional. If the network is in custom subnet mode, then this field is required. Example: `cf-east`. -* **ephemeral\_external\_ip** (Boolean, optional) - If instances must have an [ephemeral external IP](https://cloud.google.com/compute/docs/instances-and-network#externaladdresses) (`false` by default). Can be overridden in resource_pools. Example: `false`. -* **ip\_forwarding** (Boolean, optional) - If instances must have [IP forwarding](https://cloud.google.com/compute/docs/networking#canipforward) enabled (`false` by default). Can be overridden in resource_pools. Example: `false`. -* **tags** (Array<String>, optional) - A list of [tags](https://cloud.google.com/compute/docs/instances/managing-instances#tags) to apply to the instances, useful if you want to apply firewall or routes rules based on tags. Will be merged with tags in resource_pools. Example: `["foo","bar"]`. +- **network\_name** (String, optional) - The name of the [Google Compute Engine Network](https://cloud.google.com/compute/docs/networking#networks) the CPI will use when creating the instance (if not set, by default it will use the `default` network). Example: `cf`. +- **xpn\_host\_project\_id** (String, optional) - The [project id](https://support.google.com/cloud/answer/6158840?hl=en) that owns the network resource to support [Shared VPC Networks (XPN)](https://cloud.google.com/compute/docs/xpn/) (if not set, it will default to the project hosting the compute resources). Example: `my-other-project`. +- **subnetwork\_name** (String, optional) - The name of the [Google Compute Engine Subnet Network](https://cloud.google.com/compute/docs/networking#subnet_network) the CPI will use when creating the instance. If the network is in legacy mode, do not provide this property. If the network is in auto subnet mode, providing the subnetwork is optional. If the network is in custom subnet mode, then this field is required. Example: `cf-east`. +- **ephemeral\_external\_ip** (Boolean, optional) - If instances must have an [ephemeral external IP](https://cloud.google.com/compute/docs/instances-and-network#externaladdresses) (`false` by default). Can be overridden in resource_pools. Example: `false`. +- **ip\_forwarding** (Boolean, optional) - If instances must have [IP forwarding](https://cloud.google.com/compute/docs/networking#canipforward) enabled (`false` by default). Can be overridden in resource_pools. Example: `false`. +- **tags** (Array<String>, optional) - A list of [tags](https://cloud.google.com/compute/docs/instances/managing-instances#tags) to apply to the instances, useful if you want to apply firewall or routes rules based on tags. Will be merged with tags in resource_pools. Example: `["foo","bar"]`. Example of manual network: @@ -69,30 +72,31 @@ networks: ``` --- + ## VM Types / VM Extensions {: #vm-types } Schema for `cloud_properties` section: -* **machine\_type** (String, required) - The name of the [Google Compute Engine Machine Type](https://cloud.google.com/compute/docs/machine-types) the CPI will use when creating the instance (required if not using `cpu` and `ram`). Example: `n1-standard-1`. -* **machine\_series** (String, optional) - The name of the [Google Compute Engine Machine Series](https://cloud.google.com/compute/docs/machine-resource) the CPI will use when creating the instance with `cpu` and `ram` (`n1` by default). Example: `n2`. -* **cpu** (Integer, required) - Number of vCPUs ([Google Compute Engine Custom Machine Types](https://cloud.google.com/custom-machine-types/)) the CPI will use when creating the instance (required if not using `machine_type`). Example: `2`. -* **ram** (Integer, required) - Amount of memory ([Google Compute Engine Custom Machine Types](https://cloud.google.com/custom-machine-types/)) the CPI will use when creating the instance (required if not using `machine_type`). Example: `2048`. -* **zone** (String, optional) - The name of the [Google Compute Engine Zone](https://cloud.google.com/compute/docs/zones) where the instance must be created. Example: `us-west1-a`. -* **root\_disk\_size\_gb** (Integer, optional) - The size (in Gb) of the instance root disk (default is `10Gb`). Example: `10`. -* **root\_disk\_type** (String, optional) - The name of the [Google Compute Engine Disk Type](https://cloud.google.com/compute/docs/disks/#overview) the CPI will use when creating the instance root disk. Example: `pd-standard`. -* **ephemeral\_disk\_type** (String, optional) - The name of the [Google Compute Engine Disk Type](https://cloud.google.com/compute/docs/disks/#overview) the CPI will use when creating the instance ephemeral disk. Currently only `local-ssd` is supported. -* **automatic\_restart** (Boolean, optional) - If the instances should be [restarted automatically](https://cloud.google.com/compute/docs/instances/setting-instance-scheduling-options#autorestart) if they are terminated for non-user-initiated reasons (`false` by default). Example: `false`. -* **on\_host\_maintenance** (String, optional) - [Instance behavior](https://cloud.google.com/compute/docs/instances/setting-instance-scheduling-options#onhostmaintenance) on infrastructure maintenance that may temporarily impact instance performance (supported values are `MIGRATE` (default) or `TERMINATE`). Example: `MIGRATE`. -* **preemptible** (Boolean, optional) - If the instances should be [preemptible](https://cloud.google.com/preemptible-vms/) (`false` by default). Example: `false`. -* **service\_account** (String, optional) - The full service account address of the service account to launch the VM with. If a value is provided, `service_scopes` will default to `https://www.googleapis.com/auth/cloud-platform` unless it is explicitly set. See [service account permissions](https://cloud.google.com/compute/docs/access/service-accounts#service_account_permissions) for more details. To use the default service account, leave this field empty and specify `service_scopes`. Example: `service-account-name@project-name.iam.gserviceaccount.com`. -* **service\_scopes** (Array<String>, optional) - If this value is specified and `service_account` is empty, `default` will be used for `service_account`. This value supports both short (e.g., `cloud-platform`) and fully-qualified (e.g., `https://www.googleapis.com/auth/cloud-platform` formats. See [Authorization scope names](https://cloud.google.com/docs/authentication#oauth_scopes) for more details. Example: `cloud-platform`. -* **accelerators** (Array<String>, optional) - The name of accelerators that will be attached to instance. You can find them by running `gcloud compute accelerator-types list`. Example: `nvidia-tesla-t4`. -* **target\_pool** (String, optional) - The name of the [Google Compute Engine Target Pool](https://cloud.google.com/compute/docs/load-balancing/network/target-pools) the instances should be added to. Example: `cf-router`. -* **backend\_service** (String OR Map<String,String>, optional) - The name of the [Google Compute Engine Backend Service](https://cloud.google.com/compute/docs/load-balancing/http/backend-service) the instances should be added to. The backend service must already be configured with an [Instance Group](https://cloud.google.com/compute/docs/instance-groups/#unmanaged_instance_groups) in the same zone as this instance. To set up [Internal Load Balancing](https://cloud.google.com/compute/docs/load-balancing/internal/) use a map and set `scheme` to `INTERNAL` and `name` to the name of the backend service. Example: `cf-router` (external), `{name: "cf-internal", scheme: "INTERNAL"} (internal)`. -* **ephemeral\_external\_ip** (Boolean, optional) - Overrides the equivalent option in the networks section. Example: `false`. -* **ip\_forwarding** (Boolean, optional) - Overrides the equivalent option in the networks section. Example: `false`. -* **tags** (Array<String>, optional) - Merged with tags from the networks section. Example: `["foo","bar"]`. -* **labels** (Map<String,String>, optional) - A dictionary of (key,value) labels applied to the VM. Example: `{"foo":"bar"}`. +- **machine\_type** (String, required) - The name of the [Google Compute Engine Machine Type](https://cloud.google.com/compute/docs/machine-types) the CPI will use when creating the instance (required if not using `cpu` and `ram`). Example: `n1-standard-1`. +- **machine\_series** (String, optional) - The name of the [Google Compute Engine Machine Series](https://cloud.google.com/compute/docs/machine-resource) the CPI will use when creating the instance with `cpu` and `ram` (`n1` by default). Example: `n2`. +- **cpu** (Integer, required) - Number of vCPUs ([Google Compute Engine Custom Machine Types](https://cloud.google.com/custom-machine-types/)) the CPI will use when creating the instance (required if not using `machine_type`). Example: `2`. +- **ram** (Integer, required) - Amount of memory ([Google Compute Engine Custom Machine Types](https://cloud.google.com/custom-machine-types/)) the CPI will use when creating the instance (required if not using `machine_type`). Example: `2048`. +- **zone** (String, optional) - The name of the [Google Compute Engine Zone](https://cloud.google.com/compute/docs/zones) where the instance must be created. Example: `us-west1-a`. +- **root\_disk\_size\_gb** (Integer, optional) - The size (in Gb) of the instance root disk (default is `10Gb`). Example: `10`. +- **root\_disk\_type** (String, optional) - The name of the [Google Compute Engine Disk Type](https://cloud.google.com/compute/docs/disks/#overview) the CPI will use when creating the instance root disk. Example: `pd-standard`. +- **ephemeral\_disk\_type** (String, optional) - The name of the [Google Compute Engine Disk Type](https://cloud.google.com/compute/docs/disks/#overview) the CPI will use when creating the instance ephemeral disk. Currently only `local-ssd` is supported. +- **automatic\_restart** (Boolean, optional) - If the instances should be [restarted automatically](https://cloud.google.com/compute/docs/instances/setting-instance-scheduling-options#autorestart) if they are terminated for non-user-initiated reasons (`false` by default). Example: `false`. +- **on\_host\_maintenance** (String, optional) - [Instance behavior](https://cloud.google.com/compute/docs/instances/setting-instance-scheduling-options#onhostmaintenance) on infrastructure maintenance that may temporarily impact instance performance (supported values are `MIGRATE` (default) or `TERMINATE`). Example: `MIGRATE`. +- **preemptible** (Boolean, optional) - If the instances should be [preemptible](https://cloud.google.com/preemptible-vms/) (`false` by default). Example: `false`. +- **service\_account** (String, optional) - The full service account address of the service account to launch the VM with. If a value is provided, `service_scopes` will default to `https://www.googleapis.com/auth/cloud-platform` unless it is explicitly set. See [service account permissions](https://cloud.google.com/compute/docs/access/service-accounts#service_account_permissions) for more details. To use the default service account, leave this field empty and specify `service_scopes`. Example: `service-account-name@project-name.iam.gserviceaccount.com`. +- **service\_scopes** (Array<String>, optional) - If this value is specified and `service_account` is empty, `default` will be used for `service_account`. This value supports both short (e.g., `cloud-platform`) and fully-qualified (e.g., `https://www.googleapis.com/auth/cloud-platform` formats. See [Authorization scope names](https://cloud.google.com/docs/authentication#oauth_scopes) for more details. Example: `cloud-platform`. +- **accelerators** (Array<String>, optional) - The name of accelerators that will be attached to instance. You can find them by running `gcloud compute accelerator-types list`. Example: `nvidia-tesla-t4`. +- **target\_pool** (String, optional) - The name of the [Google Compute Engine Target Pool](https://cloud.google.com/compute/docs/load-balancing/network/target-pools) the instances should be added to. Example: `cf-router`. +- **backend\_service** (String OR Map<String,String>, optional) - The name of the [Google Compute Engine Backend Service](https://cloud.google.com/compute/docs/load-balancing/http/backend-service) the instances should be added to. The backend service must already be configured with an [Instance Group](https://cloud.google.com/compute/docs/instance-groups/#unmanaged_instance_groups) in the same zone as this instance. To set up [Internal Load Balancing](https://cloud.google.com/compute/docs/load-balancing/internal/) use a map and set `scheme` to `INTERNAL` and `name` to the name of the backend service. Example: `cf-router` (external), `{name: "cf-internal", scheme: "INTERNAL"} (internal)`. +- **ephemeral\_external\_ip** (Boolean, optional) - Overrides the equivalent option in the networks section. Example: `false`. +- **ip\_forwarding** (Boolean, optional) - Overrides the equivalent option in the networks section. Example: `false`. +- **tags** (Array<String>, optional) - Merged with tags from the networks section. Example: `["foo","bar"]`. +- **labels** (Map<String,String>, optional) - A dictionary of (key,value) labels applied to the VM. Example: `{"foo":"bar"}`. Example of an `n1-standard-2` VM: @@ -146,11 +150,12 @@ instance_groups: ``` --- + ## Disk Types {: #disk-types } Schema for `cloud_properties` section: -* **type** [String, optional]: Type of the [disk](https://cloud.google.com/compute/docs/disks/#overview): `pd-standard`, `pd-ssd`. Defaults to `pd-standard`. +- **type** [String, optional]: Type of the [disk](https://cloud.google.com/compute/docs/disks/#overview): `pd-standard`, `pd-ssd`. Defaults to `pd-standard`. Persistent disks are created in the zone of a VM that disk will be attached. @@ -163,6 +168,7 @@ disk_types: ``` --- + ## Global Configuration {: #global } The CPI can only talk to a single Google Compute Engine region. @@ -170,6 +176,7 @@ The CPI can only talk to a single Google Compute Engine region. See [all configuration options](https://bosh.io/jobs/google_cpi?source=github.com/cloudfoundry/bosh-google-cpi-release). --- + ## Example Cloud Config {: #cloud-config } ```yaml diff --git a/content/google-required-permissions.md b/content/google-required-permissions.md index 6d6469900..8297787e4 100644 --- a/content/google-required-permissions.md +++ b/content/google-required-permissions.md @@ -1,3 +1,5 @@ +# Google - Required Permissions + This topic describes how to configure BOSH with the minimum set of permissions on Google Cloud Engine. ## Google CPI and Director {: #bosh-director } @@ -119,12 +121,12 @@ In addition to the above permissions, you will need to add the following permiss When using the `service_account` or `service_scopes` properties, you will need to add: - - `compute.instances.setServiceAccount` permission - - `iam.serviceAccountUser` role +- `compute.instances.setServiceAccount` permission +- `iam.serviceAccountUser` role When using the `accelerators` property, you will need to add: - - `compute.acceleratorTypes.get` +- `compute.acceleratorTypes.get` ## Minimum permissions for GCS blobstore {: #director-with-gcs-blobstore } @@ -167,7 +169,7 @@ This configuration is similar to the previous one except that it's used when the !!! note The agent does not need to delete files from the blobstore -4. Configure roles. +3. Configure roles. ```shell gcloud beta iam roles --project create blobstore.director \ @@ -177,9 +179,9 @@ This configuration is similar to the previous one except that it's used when the --file <( bosh int -v project_id= agent-blobstore-role.yml ) ``` -5. On Google Cloud navigate go to `IAM & admin` > `Service accounts` and click on `CREATE SERVICE ACCOUNT`. +4. On Google Cloud navigate go to `IAM & admin` > `Service accounts` and click on `CREATE SERVICE ACCOUNT`. Give your service account a name, check `Furnish a new private key` and select the aforementioned roles. ![image](images/gcp-service-account.png) -6. Configure [GCS Blobstore](director-configure-blobstore.md#gcs) +5. Configure [GCS Blobstore](director-configure-blobstore.md#gcs) diff --git a/content/google.md b/content/google.md index d00beb382..d4e525130 100644 --- a/content/google.md +++ b/content/google.md @@ -1,32 +1,26 @@ ---- -title: Google Cloud Platform ---- - # Google Cloud Platform The `google` CPI can be used with [Google Cloud Platform](https://cloud.google.com/). - * Release: [cloudfoundry/bosh-google-cpi-release](https://github.com/cloudfoundry/bosh-google-cpi-release) - * Issues: [GitHub Issues](https://github.com/cloudfoundry/bosh-google-cpi-release/issues) - * Slack: [cloudfoundry#bosh-gce-cpi](https://cloudfoundry.slack.com/messages/bosh-gce-cpi) - +- Release: [cloudfoundry/bosh-google-cpi-release](https://github.com/cloudfoundry/bosh-google-cpi-release) +- Issues: [GitHub Issues](https://github.com/cloudfoundry/bosh-google-cpi-release/issues) +- Slack: [cloudfoundry#bosh-gce-cpi](https://cloudfoundry.slack.com/messages/bosh-gce-cpi) ## Concepts The following table maps BOSH concepts to their respective IaaS concept. -| BOSH | Google Cloud Platform | -| ----------------- | --------------------- | -| Availability Zone | [Zone](https://cloud.google.com/compute/docs/regions-zones/) | -| Virtual Machine | [Virtual Machine Instance](https://cloud.google.com/compute/docs/instances/) | -| Network Subnet | [VPC Subnet](https://cloud.google.com/vpc/docs/vpc#vpc_networks_and_subnets) | -| Virtual IP | [Static External IP](https://cloud.google.com/compute/docs/ip-addresses/#reservedaddress) | -| Persistent Disk | [Persistent Disks](https://cloud.google.com/persistent-disk/) | -| Disk Snapshot | [Persistent Disk Snapshots](https://cloud.google.com/compute/docs/disks/create-snapshots) | -| Stemcell | [Compute Custom Image](https://cloud.google.com/compute/docs/images#custom_images) | +| BOSH | Google Cloud Platform | +| ----------------- | ------------------------------------------------------------------------------------------------------------------- | +| Availability Zone | [Zone](https://cloud.google.com/compute/docs/regions-zones/) | +| Virtual Machine | [Virtual Machine Instance](https://cloud.google.com/compute/docs/instances/) | +| Network Subnet | [VPC Subnet](https://cloud.google.com/vpc/docs/vpc#vpc_networks_and_subnets) | +| Virtual IP | [Static External IP](https://cloud.google.com/compute/docs/ip-addresses/#reservedaddress) | +| Persistent Disk | [Persistent Disks](https://cloud.google.com/persistent-disk/) | +| Disk Snapshot | [Persistent Disk Snapshots](https://cloud.google.com/compute/docs/disks/create-snapshots) | +| Stemcell | [Compute Custom Image](https://cloud.google.com/compute/docs/images#custom_images) | | Agent Settings | [Instance Custom Metadata](https://cloud.google.com/compute/docs/storing-retrieving-metadata) (`bosh_settings` key) | - ## Feature Support The following sections describe some specific BOSH features supported by the diff --git a/content/guide-ipv6-on-vsphere.md b/content/guide-ipv6-on-vsphere.md index 564fa9175..39c411521 100644 --- a/content/guide-ipv6-on-vsphere.md +++ b/content/guide-ipv6-on-vsphere.md @@ -1,3 +1,5 @@ +# Using IPv6 on vSphere + !!! note BOSH supports IPv6 on vSphere since version bosh-release v264+, stemcell 3468.11+ and CLI v2.0.45+. diff --git a/content/guide-multi-cpi-aws.md b/content/guide-multi-cpi-aws.md index b2b9d26c2..0f7b710bb 100644 --- a/content/guide-multi-cpi-aws.md +++ b/content/guide-multi-cpi-aws.md @@ -1,3 +1,5 @@ +# Multi-CPI on AWS + !!! note BOSH supports Multi-CPI since version v261+. @@ -6,6 +8,7 @@ In this guide we explore how to configure BOSH to deploy VMs from a single deplo For simplicity reasons we're going to allow all internal traffic between two VPCs, however this can be configured as desired by the operator. --- + ## Set up the IaaS {: #setup-iaas } Let's start by initializing main AZ (`z1`) to US East (N. Virginia) by following steps 1 and 2 from [Creating environment on AWS](init-aws.md). This will give you a working BOSH Director in a single region. You can perform a deployment to test Director is working fine. @@ -13,17 +16,17 @@ Let's start by initializing main AZ (`z1`) to US East (N. Virginia) by following To add a second AZ (`z2`) to US West (N. California) you need to perform step 1 from [Creating environment on AWS](init-aws.md) in another AWS account. --- + ## Connecting VPCs {: #connecting-vpcs } The VMs in one AZ need to be able to talk to VMs in the other AZ. We're going to describe two ways AZs can be connected. You have two options: - if VPCs are in the same AWS region you can simply use [VPC Peering](guide-multi-cpi-aws.md#vpc-peering) as shown below - - if VPCs are in different regions you will need to connect them through a [OpenVPN](guide-multi-cpi-aws.md#openvpn) as shown below - - if VPCs are spread out across multiple regions, you can mix and match two approaches above --- + ### VPC Peering (only works for VPCs in the same region) {: #vpc-peering } To connect VPCs in the same region you have to create a VPC Peering Connection between each region. In our case, we have two VPCs so only one connection is required. @@ -50,13 +53,14 @@ To connect VPCs in the same region you have to create a VPC Peering Connection b If you want IPv6 traffic to be routed you also need to add the corresponding IPv6 CIDR blocks. --- + ### OpenVPN using IPSec {: #openvpn } Here we are going to use the [OpenVPN BOSH Release](https://github.com/dpb587/openvpn-bosh-release) to connect both OpenVPN Server and client in each region like shown below: ![image](images/multi-cpi/aws-iaas-topology.png) -0. Setup local Multi-CPI directories: +1. Setup local Multi-CPI directories: ```shell mkdir -p ~/workspace/multi-cpi-vpn @@ -68,9 +72,9 @@ Here we are going to use the [OpenVPN BOSH Release](https://github.com/dpb587/op cd multi-cpi-vpn ``` -0. Allocate Elastic IPs for each VPN Server in their respective regions. +1. Allocate Elastic IPs for each VPN Server in their respective regions. -0. Create following files `~/workspace/multi-cpi-vpn/creds-az1.yml` and `~/workspace/multi-cpi-vpn/creds-az2.yml` with the following properties. You should have all this information from the above [Set up the IaaS](guide-multi-cpi-aws.md#setup-iaas) step. +1. Create following files `~/workspace/multi-cpi-vpn/creds-az1.yml` and `~/workspace/multi-cpi-vpn/creds-az2.yml` with the following properties. You should have all this information from the above [Set up the IaaS](guide-multi-cpi-aws.md#setup-iaas) step. ```yaml access_key_id: @@ -85,7 +89,7 @@ Here we are going to use the [OpenVPN BOSH Release](https://github.com/dpb587/op route_table_id: # e.g. rtb-4127673b ``` -0. Generate certificates for each server and client. +1. Generate certificates for each server and client. ```shell bosh int ~/workspace/bosh-multi-cpi-kb/templates/vpn-ca.yml \ @@ -97,7 +101,7 @@ Here we are going to use the [OpenVPN BOSH Release](https://github.com/dpb587/op --vars-store=~/workspace/multi-cpi-vpn/certs-vpn-az2.yml ``` -0. Deploy OpenVPN Servers in each AZ. +1. Deploy OpenVPN Servers in each AZ. ```shell # Create VPN server in z1 @@ -148,6 +152,7 @@ Here we are going to use the [OpenVPN BOSH Release](https://github.com/dpb587/op ``` --- + ## Configure CPI and Cloud configs {: #configuring-configs } Now that the IaaS is configured, update your Director's [CPI config](cpi-config.md): @@ -229,6 +234,7 @@ bosh update-cloud-config cloud.yml ``` --- + ## Deploy example Zookeeper deployment {: #deploying } ... diff --git a/content/hm-config.md b/content/hm-config.md index 17ae9619c..cd1a9e478 100644 --- a/content/hm-config.md +++ b/content/hm-config.md @@ -1,11 +1,15 @@ +# Using the Health Monitor + Sections below only show minimum configuration options to enable plugins. Add them to the deployment manifest for the Health Monitor. See [health_monitor release job properties](http://bosh.io/jobs/health_monitor?source=github.com/cloudfoundry/bosh) for more details. --- + ## Event Logger {: #logger } Enabled by default. No way to turn it off. --- + ## Resurrector {: #resurrector } Restarts VMs that have stopped heartbeating. See [Automatic repair with Resurrector](resurrector.md) for more details. @@ -17,6 +21,7 @@ properties: ``` --- + ## Emailer {: #emailer } Plugin that sends configurable e-mails on events received. @@ -38,6 +43,7 @@ properties: ``` --- + ## JSON {: #json } Enabled by default. @@ -45,6 +51,7 @@ Enabled by default. Plugin that sends alerts and heartbeats as json to programs installed on the director over stdin. The plugin will start and manage a process for each executable matching the glob `/var/vcap/jobs/*/bin/bosh-monitor/*`. --- + ## OpenTSDB {: #tsdb } Plugin that forwards alerts and heartbeats to [OpenTSDB](http://opentsdb.net/). @@ -59,6 +66,7 @@ properties: ``` --- + ## Graphite {: #graphite } Plugin that forwards heartbeats to [Graphite](https://graphite.readthedocs.org/en/latest/). @@ -73,6 +81,7 @@ properties: ``` --- + ## PagerDuty {: #pagerduty } Plugin that sends various events to [PagerDuty.com](http://pagerduty.com) using their API. @@ -87,6 +96,7 @@ properties: ``` --- + ## DataDog {: #datadog } Plugin that sends various events to [DataDog.com](http://datadoghq.com) using their API. diff --git a/content/index.md b/content/index.md index 18f492f31..fa08d810d 100644 --- a/content/index.md +++ b/content/index.md @@ -6,10 +6,8 @@ BOSH is a project that unifies release engineering, deployment, and lifecycle ma While BOSH was developed to deploy Cloud Foundry PaaS, it can also be used to deploy almost any other software (Hadoop, for instance). BOSH is particularly well-suited for large distributed systems. In addition, BOSH supports multiple Infrastructure as a Service (IaaS) providers like VMware vSphere, Google Cloud Platform, Amazon Web Services EC2, Microsoft Azure, OpenStack, and Alibaba Cloud. There is a Cloud Provider Interface (CPI) that enables users to extend BOSH to support additional IaaS providers such as Apache CloudStack and VirtualBox. - ## Getting Started ### CLI The [`bosh` CLI](cli-v2.md) is the command line tool used for interacting with all things BOSH. Release binaries are available on [GitHub](https://github.com/cloudfoundry/bosh-cli/releases). See [Installation](cli-v2-install.md) for more details on how to download and install. - diff --git a/content/init-alicloud.md b/content/init-alicloud.md index f1d4da0dc..0a5838186 100644 --- a/content/init-alicloud.md +++ b/content/init-alicloud.md @@ -1,3 +1,5 @@ +# Init Alibaba Cloud (AliCloud) Environment + This document shows how to set up new [environment](terminology.md#environment) on Alibaba Cloud (AliCloud) ## Step 1: Prepare an Alibaba Cloud Account {: #prepare-alicloud } @@ -6,18 +8,20 @@ If you do not have an Alibaba Cloud account, [create one](https://account.alibab To configure your Alibaba Cloud account: -* [Obtain Alibaba Cloud credentials](#credentials) -* [Create a Virtual Private Cloud (VPC)](#create-vpc) -* [Create an Elastic IP](#create-eip) -* [Create a Key Pair](#create-key-pair) -* [Create and Configure Security Group](#create-security) +- [Obtain Alibaba Cloud credentials](#credentials) +- [Create a Virtual Private Cloud (VPC)](#create-vpc) +- [Create an Elastic IP](#create-eip) +- [Create a Key Pair](#create-key-pair) +- [Create and Configure Security Group](#create-security) --- + ### Obtain Alibaba Cloud Credentials {: #credentials } Your Alibaba Cloud credentials consist of an Access Key ID and a Secret Access Key. Follow [Creating RAM Users](https://www.alibabacloud.com/help/doc-detail/28647.htm) to create a new RAM user. --- + ### Create a Virtual Private Cloud (VPC) {: #create-vpc } 1. Log on to the [VPC console](https://vpcnext.console.aliyun.com). @@ -29,6 +33,7 @@ Your Alibaba Cloud credentials consist of an Access Key ID and a Secret Access K See [Create a VPC](https://www.alibabacloud.com/help/doc-detail/65430.htm). --- + ### Create an Elastic IP {: #create-eip } 1. On the VPC Dashboard, click **Elastic IPs** and click **Create EIP**. @@ -37,8 +42,8 @@ See [Create a VPC](https://www.alibabacloud.com/help/doc-detail/65430.htm). See [Create an EIP](https://www.alibabacloud.com/help/doc-detail/65203.htm). - --- + ### Create a Key Pair {: #create-key-pair } 1. Log on to the [ECS console](https://ecs.console.aliyun.com). @@ -54,20 +59,20 @@ See [Create an EIP](https://www.alibabacloud.com/help/doc-detail/65203.htm). See [Create an SSH key pair](https://www.alibabacloud.com/help/doc-detail/51793.htm) --- + ### Create and Configure Security Group {: #create-security } Log on to the ECS console. In the left-side navigation pane, select Networks & Security > > Security group. - 1. On the ECS Dashboard, select **Networks & Security** and then select **Security group**. 1. Select a region and then click **Create Security Group**. 1. Complete the Create Security Group form with the following information: - * **Security group name**: bosh - * **Description**: BOSH deployed VMs - * **VPC**: Select the "bosh" VPC that you created in [Create a Virtual Private Cloud](#create-vpc). + - **Security group name**: bosh + - **Description**: BOSH deployed VMs + - **VPC**: Select the "bosh" VPC that you created in [Create a Virtual Private Cloud](#create-vpc). 1. Select the created security group with group name "bosh", in the Actions column click Configure Rules. @@ -76,29 +81,22 @@ In the left-side navigation pane, select Networks & Security > > Security group. 1. Fill out the Edit inbound rules form and click **Save**. !!! note - It highly discouraged to run any production environment with 0.0.0.0/0 source or to make any BOSH management ports publicly accessible. - - - - - - - - + It highly discouraged to run any production environment with `0.0.0.0/0` source or to make any BOSH management ports publicly accessible. - - - - - - -
TypePort RangeSourcePurpose
Custom TCP Rule22(My IP)SSH access from CLI
Custom TCP Rule6868(My IP)BOSH Agent access from CLI
Custom TCP Rule25555(My IP)BOSH Director access from CLI
All TCP0 - 65535ID of this security groupManagement and data access
All UDP0 - 65535ID of this security groupManagement and data access
+ | Type | Port Range | Source | Purpose | + |-----------------|------------|---------------------------|-------------------------------| + | Custom TCP Rule | 22 | (My IP) | SSH access from CLI | + | Custom TCP Rule | 6868 | (My IP) | BOSH Agent access from CLI | + | Custom TCP Rule | 25555 | (My IP) | BOSH Director access from CLI | + | All TCP | 0 - 65535 | ID of this security group | Management and data access | + | All UDP | 0 - 65535 | ID of this security group | Management and data access | See [Creating a Security Group](https://www.alibabacloud.com/help/doc-detail/25468.htm) See [Add security group rules](https://www.alibabacloud.com/help/doc-detail/25471.htm) --- + ## Step 2: Deploy {: #deploy } 1. Install [CLI v2](cli-v2.md). diff --git a/content/init-aws.md b/content/init-aws.md index ce163cb33..0061d396c 100644 --- a/content/init-aws.md +++ b/content/init-aws.md @@ -1,3 +1,5 @@ +# Init AWS Environment + This document shows how to set up new [environment](terminology.md#environment) on Amazon Web Services (AWS). ## Step 1: Prepare an AWS Account {: #prepare-aws } @@ -6,18 +8,20 @@ If you do not have an AWS account, [create one](http://goo.gl/MaAybK). To configure your AWS account: -* [Obtain AWS credentials](#credentials) -* [Create a Virtual Private Cloud (VPC)](#create-vpc) -* [Create an Elastic IP](#create-eip) -* [Create a Key Pair](#create-key-pair) -* [Create and Configure Security Group](#create-security) +- [Obtain AWS credentials](#credentials) +- [Create a Virtual Private Cloud (VPC)](#create-vpc) +- [Create an Elastic IP](#create-eip) +- [Create a Key Pair](#create-key-pair) +- [Create and Configure Security Group](#create-security) --- + ### Obtain AWS Credentials {: #credentials } Your AWS credentials consist of an Access Key ID and a Secret Access Key. Follow [Creating IAM Users](aws-iam-users.md#create) to create a new IAM user. --- + ### Create a Virtual Private Cloud (VPC) {: #create-vpc } 1. In the upper-right corner of the AWS Console, select a Region. @@ -35,13 +39,13 @@ Your AWS credentials consist of an Access Key ID and a Secret Access Key. Follow 1. Select **VPC with a Single Public Subnet** and click **Select**. 1. Complete the VPC form with the following information: - * **IP CIDR block**: 10.0.0.0/16 - * **VPC name**: bosh - * **Public subnet**: 10.0.0.0/24 - * **Availability Zone**: us-east-1a - * **Subnet name**: public - * **Enable DNS hostnames**: Yes - * **Hardware tenancy**: Default + - **IP CIDR block**: 10.0.0.0/16 + - **VPC name**: bosh + - **Public subnet**: 10.0.0.0/24 + - **Availability Zone**: us-east-1a + - **Subnet name**: public + - **Enable DNS hostnames**: Yes + - **Hardware tenancy**: Default ![image](images/deploy-microbosh-to-aws/create-vpc.png) @@ -52,6 +56,7 @@ Your AWS credentials consist of an Access Key ID and a Secret Access Key. Follow ![image](images/deploy-microbosh-to-aws/list-subnets.png) --- + ### Create an Elastic IP {: #create-eip } 1. On the VPC Dashboard, click **Elastic IPs** and click **Allocate New Address**. @@ -65,6 +70,7 @@ Your AWS credentials consist of an Access Key ID and a Secret Access Key. Follow ![image](images/deploy-microbosh-to-aws/list-elastic-ips.png) --- + ### Create a Key Pair {: #create-key-pair } 1. In the AWS Console, select **EC2** to get to the EC2 Dashboard. @@ -80,6 +86,7 @@ Your AWS credentials consist of an Access Key ID and a Secret Access Key. Follow 1. Save private key to `~/Downloads/bosh.pem`. --- + ### Create and Configure Security Group {: #create-security } 1. On the EC2 Dashboard, click **Security Groups** and then click **Create Security Group**. @@ -87,9 +94,9 @@ Your AWS credentials consist of an Access Key ID and a Secret Access Key. Follow ![image](images/deploy-microbosh-to-aws/list-security-groups.png) 1. Complete the Create Security Group form with the following information: - * **Security group name**: bosh - * **Description**: BOSH deployed VMs - * **VPC**: Select the "bosh" VPC that you created in [Create a Virtual Private Cloud](#create-vpc). + - **Security group name**: bosh + - **Description**: BOSH deployed VMs + - **VPC**: Select the "bosh" VPC that you created in [Create a Virtual Private Cloud](#create-vpc). 1. Click **Create** @@ -102,23 +109,15 @@ Your AWS credentials consist of an Access Key ID and a Secret Access Key. Follow 1. Fill out the Edit inbound rules form and click **Save**. !!! note - It highly discouraged to run any production environment with 0.0.0.0/0 source or to make any BOSH management ports publicly accessible. - - - - - - - - + It highly discouraged to run any production environment with `0.0.0.0/0` source or to make any BOSH management ports publicly accessible. - - - - - - -
TypePort RangeSourcePurpose
Custom TCP Rule22(My IP)SSH access from CLI
Custom TCP Rule6868(My IP)BOSH Agent access from CLI
Custom TCP Rule25555(My IP)BOSH Director access from CLI
All TCP0 - 65535ID of this security groupManagement and data access
All UDP0 - 65535ID of this security groupManagement and data access
+ | Type | Port Range | Source | Purpose | + |-----------------|------------|---------------------------|-------------------------------| + | Custom TCP Rule | 22 | (My IP) | SSH access from CLI | + | Custom TCP Rule | 6868 | (My IP) | BOSH Agent access from CLI | + | Custom TCP Rule | 25555 | (My IP) | BOSH Director access from CLI | + | All TCP | 0 - 65535 | ID of this security group | Management and data access | + | All UDP | 0 - 65535 | ID of this security group | Management and data access | !!! note To enter your security group as a *Source*, select *Custom IP*, and enter "bosh". Note: The AWS Console should autocomplete the security group ID (e.g. "sg-12ab34cd"). @@ -126,6 +125,7 @@ Your AWS credentials consist of an Access Key ID and a Secret Access Key. Follow ![image](images/deploy-microbosh-to-aws/edit-security-group-rules.png) --- + ## Step 2: Deploy {: #deploy } 1. Install [CLI v2](cli-v2.md). diff --git a/content/init-azure.md b/content/init-azure.md index 10d410c6e..569c6c896 100644 --- a/content/init-azure.md +++ b/content/init-azure.md @@ -1,3 +1,5 @@ +# Init Azure Environment + This document shows how to initialize new [environment](terminology.md#environment) on Microsoft Azure. ## Step 1: Prepare an Azure Environment {: #prepare } @@ -11,6 +13,7 @@ We strongly recommend you to use Azure template [bosh-setup](https://github.com/ To prepare your Azure environment find out and/or create any missing resources in Azure. If you are not familiar with Azure take a look at [Creating Azure resources](azure-resources.md) page for more details on how to create and configure necessary resources: --- + ## Step 2: Deploy {: #deploy } 1. Install [CLI v2](cli-v2.md). diff --git a/content/init-external-ip.md b/content/init-external-ip.md index 613ea1ccd..8720222c2 100644 --- a/content/init-external-ip.md +++ b/content/init-external-ip.md @@ -1,3 +1,5 @@ +# Init External IP + It's strongly recommended to not allow ingress traffic to the Director VM via public IP. One way to achieve that is to use a [jumpbox](terminology.md#jumpbox) to access internal networks. If you do have a jumpbox consider using [CLI tunneling functionality](cli-tunnel.md) instead of running CLI from the jumpbox VM. diff --git a/content/init-google.md b/content/init-google.md index bd371b7fa..d4986b33f 100644 --- a/content/init-google.md +++ b/content/init-google.md @@ -1,3 +1,5 @@ +# Init Google Environment + This document shows how to initialize new [environment](terminology.md#environment) on Google Cloud Platform. 1. Install [CLI v2](cli-v2.md). diff --git a/content/init-openstack.md b/content/init-openstack.md index a9ea35898..69d3ba9ec 100644 --- a/content/init-openstack.md +++ b/content/init-openstack.md @@ -1,3 +1,5 @@ +# Init OpenStack Environment + This document shows how to initialize new [environment](terminology.md#environment) on OpenStack. ## Step 1: Prepare an OpenStack environment {: #prepare-openstack } @@ -7,20 +9,20 @@ This document shows how to initialize new [environment](terminology.md#environme 1. An OpenStack environment running one of the supported releases. See [bosh-openstack-cpi](https://github.com/cloudfoundry/bosh-openstack-cpi-release#supported-openstack-versions) for information on the CPI's support policy. 1. The following OpenStack services: - * [Identity](https://www.openstack.org/software/releases/ocata/components/keystone): + - [Identity](https://www.openstack.org/software/releases/ocata/components/keystone): BOSH authenticates credentials and retrieves the endpoint URLs for other OpenStack services. - * [Compute](https://www.openstack.org/software/releases/ocata/components/nova): + - [Compute](https://www.openstack.org/software/releases/ocata/components/nova): BOSH boots new VMs, assigns floating IPs to VMs, and creates and attaches volumes to VMs. - * [Image](https://www.openstack.org/software/releases/ocata/components/glance): + - [Image](https://www.openstack.org/software/releases/ocata/components/glance): BOSH stores stemcells using the Image service. - * **(Optional)** [OpenStack Networking](https://www.openstack.org/software/releases/ocata/components/neutron): + - **(Optional)** [OpenStack Networking](https://www.openstack.org/software/releases/ocata/components/neutron): Provides network scaling and automated management functions that are useful when deploying complex distributed systems. **Note:** OpenStack networking is used as default as of v28 of the OpenStack CPI. To disable the use of the OpenStack Networking project, see [Customize the Deployment](#customize-deployment). 1. Access to an existing OpenStack project. 1. The following OpenStack networks: - * An external network with a subnet, that can assign a floating IP. - * A private network with a subnet. The subnet must have an IP address allocation pool. Will be created by Terraform automatically. + - An external network with a subnet, that can assign a floating IP. + - A private network with a subnet. The subnet must have an IP address allocation pool. Will be created by Terraform automatically. 1. OpenStack flavor `m1.xlarge` The flavor is hard coded in `bosh-deployment/openstack/cpi.yml`. @@ -43,11 +45,12 @@ Applying the Terraform plan to the environment will output the variables necessa Instead of using Terraform, you can do the following things manually as described below: -* Create a [Keypair](#keypair). -* Create and configure [Security Groups](#security-groups). -* Allocate a [floating IP address](#floating-ip). +- Create a [Keypair](#keypair). +- Create and configure [Security Groups](#security-groups). +- Allocate a [floating IP address](#floating-ip). --- + ##### Create a Keypair {: #keypair } 1. Select **Access & Security** from the left navigation panel. @@ -67,6 +70,7 @@ Instead of using Terraform, you can do the following things manually as describe ![image](images/micro-openstack/save-keypair.png) --- + ##### Create and Configure BOSH Security Group {: #security-groups } You must create and configure a security group to restrict incoming network traffic to the BOSH VMs. @@ -96,27 +100,17 @@ You must create and configure a security group to restrict incoming network traf !!! warning It highly discouraged to run any production environment with `0.0.0.0/0` source or to make any BOSH management ports publicly accessible. - - - - - - - - - - - - - - - - - - -
DirectionEther TypeIP ProtocolPort RangeRemotePurpose
IngressIPv4TCP220.0.0.0/0 (CIDR)SSH access from CLI
IngressIPv4TCP68680.0.0.0/0 (CIDR)BOSH Agent access from CLI
IngressIPv4TCP255550.0.0.0/0 (CIDR)BOSH Director access from CLI
EgressIPv4Any-0.0.0.0/0 (CIDR)
EgressIPv6Any-::/0 (CIDR)
IngressIPv4TCP1-65535boshManagement and data access
+ | Direction | Ether Type | IP Protocol | Port Range | Remote | Purpose | + |-----------|------------|-------------|------------|------------------|-------------------------------| + | Ingress | IPv4 | TCP | 22 | 0.0.0.0/0 (CIDR) | SSH access from CLI | + | Ingress | IPv4 | TCP | 6868 | 0.0.0.0/0 (CIDR) | BOSH Agent access from CLI | + | Ingress | IPv4 | TCP | 25555 | 0.0.0.0/0 (CIDR) | BOSH Director access from CLI | + | Egress | IPv4 | Any | - | 0.0.0.0/0 (CIDR) | | + | Egress | IPv6 | Any | - | ::/0 (CIDR) | | + | Ingress | IPv4 | TCP | 1-65535 | bosh | Management and data access | --- + ##### Allocate a Floating IP Address {: #floating-ip } 1. Select **Access & Security** from the left navigation panel. @@ -138,6 +132,7 @@ You must create and configure a security group to restrict incoming network traf ![image](images/micro-openstack/floating-ip.png) --- + ## Step 2: Deploy {: #deploy } ### Prerequisites @@ -181,7 +176,7 @@ If you used Terraform as described in [prerequisites](#prerequisites) you can us The variable names from the Terraform output match those in the `vars.yml`. Here is an example of the Terraform output section: -``` +```terraform default_key_name = bosh-1 default_security_groups = [bosh] external_ip = 192.168.1.19 @@ -245,16 +240,16 @@ See [OpenStack CPI errors](openstack-cpi-errors.md) for list of common errors an #### Customize the Deployment {: #customize-deployment } -* using internal DNS, i.e. if it is required to resolve the OpenStack API endpoint: [bosh-deployment/misc/dns.yml](https://github.com/cloudfoundry/bosh-deployment/blob/master/misc/dns.yml) -* using `boot-from-volume` to have nova create the boot volume as a cinder device (necessary for live-migration of VMs): [bosh-deployment/openstack/boot-from-volume.yml](https://github.com/cloudfoundry/bosh-deployment/blob/master/openstack/boot-from-volume.yml) -* using a custom CA for your OpenStack endpoints: [bosh-deployment/openstack/custom-ca.yml](https://github.com/cloudfoundry/bosh-deployment/blob/master/openstack/custom-ca.yml) -* putting additional trusted certificates into the cert-store of deployed VMs: [bosh-deployment/openstack/trusted-certs.yml](https://github.com/cloudfoundry/bosh-deployment/blob/master/openstack/trusted-certs.yml) -* using a custom ntp server for deployed VMs: [bosh-deployment/misc/ntp.yml](https://github.com/cloudfoundry/bosh-deployment/blob/master/misc/ntp.yml) -* using keystone v2 instead of keystone v3: [bosh-deployment/openstack/keystone-v2.yml](https://github.com/cloudfoundry/bosh-deployment/blob/master/openstack/keystone-v2.yml) -* enable soft anti affinity for each instance group: [bosh-deployment/openstack/auto-anti-affinity.yml](https://github.com/cloudfoundry/bosh-deployment/blob/master/openstack/auto-anti-affinity.yml) -* disable human readable VM names and use UUIDs instead: [bosh-deployment/openstack/disable-readable-vm-names.yml](https://github.com/cloudfoundry/bosh-deployment/blob/master/openstack/disable-readable-vm-names.yml) -* using nova networking instead of neutron networking: [bosh-deployment/openstack/nova-networking.yml](https://github.com/cloudfoundry/bosh-deployment/blob/master/openstack/nova-networking.yml) -* enable native CPI disk resizing: [bosh-deployment/misc/cpi-resize-disk.yml](https://github.com/cloudfoundry/bosh-deployment/blob/master/misc/cpi-resize-disk.yml) +- using internal DNS, i.e. if it is required to resolve the OpenStack API endpoint: [bosh-deployment/misc/dns.yml](https://github.com/cloudfoundry/bosh-deployment/blob/master/misc/dns.yml) +- using `boot-from-volume` to have nova create the boot volume as a cinder device (necessary for live-migration of VMs): [bosh-deployment/openstack/boot-from-volume.yml](https://github.com/cloudfoundry/bosh-deployment/blob/master/openstack/boot-from-volume.yml) +- using a custom CA for your OpenStack endpoints: [bosh-deployment/openstack/custom-ca.yml](https://github.com/cloudfoundry/bosh-deployment/blob/master/openstack/custom-ca.yml) +- putting additional trusted certificates into the cert-store of deployed VMs: [bosh-deployment/openstack/trusted-certs.yml](https://github.com/cloudfoundry/bosh-deployment/blob/master/openstack/trusted-certs.yml) +- using a custom ntp server for deployed VMs: [bosh-deployment/misc/ntp.yml](https://github.com/cloudfoundry/bosh-deployment/blob/master/misc/ntp.yml) +- using keystone v2 instead of keystone v3: [bosh-deployment/openstack/keystone-v2.yml](https://github.com/cloudfoundry/bosh-deployment/blob/master/openstack/keystone-v2.yml) +- enable soft anti affinity for each instance group: [bosh-deployment/openstack/auto-anti-affinity.yml](https://github.com/cloudfoundry/bosh-deployment/blob/master/openstack/auto-anti-affinity.yml) +- disable human readable VM names and use UUIDs instead: [bosh-deployment/openstack/disable-readable-vm-names.yml](https://github.com/cloudfoundry/bosh-deployment/blob/master/openstack/disable-readable-vm-names.yml) +- using nova networking instead of neutron networking: [bosh-deployment/openstack/nova-networking.yml](https://github.com/cloudfoundry/bosh-deployment/blob/master/openstack/nova-networking.yml) +- enable native CPI disk resizing: [bosh-deployment/misc/cpi-resize-disk.yml](https://github.com/cloudfoundry/bosh-deployment/blob/master/misc/cpi-resize-disk.yml) ### Connect to the Director diff --git a/content/init-vsphere-rp.md b/content/init-vsphere-rp.md index db03e2d5f..f87f1af68 100644 --- a/content/init-vsphere-rp.md +++ b/content/init-vsphere-rp.md @@ -1,3 +1,5 @@ +# Bootstrapping with Resource Pools + If the BOSH director is required to be deployed within a vSphere Resource Pool, utilize the following additional CLI arguments when creating the BOSH env: ```shell diff --git a/content/init-vsphere.md b/content/init-vsphere.md index 4de28d4be..79cfd6a41 100644 --- a/content/init-vsphere.md +++ b/content/init-vsphere.md @@ -1,3 +1,5 @@ +# Init vSphere Environment + This document shows how to set up new [environment](terminology.md#environment) on vSphere. 1. Install [CLI v2](cli-v2.md). diff --git a/content/init.md b/content/init.md index ad3248ac5..0d416c900 100644 --- a/content/init.md +++ b/content/init.md @@ -1,3 +1,5 @@ +# Deploying BOSH with create-env + An environment consists of the Director and deployments that it orchestrates. First we need to deploy the Director which then would be able to manage other deployments. Choose your next step where to install the Director: diff --git a/content/instance-metadata.md b/content/instance-metadata.md index 557d8e5e5..b8cc419be 100644 --- a/content/instance-metadata.md +++ b/content/instance-metadata.md @@ -1,3 +1,5 @@ +# Instance Metadata + !!! note This feature is available with bosh-release v255.4+ and on 3213+ stemcell series. @@ -13,11 +15,11 @@ Accessing the Bosh structural information over filesystem might be useful when b Each VM has a `/var/vcap/instance` directory that contains following files: -* `deployment`: Name of the deployment that the instance belongs to. -* `name`: Name of the _instance group_ that the instance belongs to. -* `az`: Name of the availability zone that the instance is placed in. -* `index`: Human-friendly ordinal for the instance within its group. -* `id`: Immutable UUID for the instance. +- `deployment`: Name of the deployment that the instance belongs to. +- `name`: Name of the _instance group_ that the instance belongs to. +- `az`: Name of the availability zone that the instance is placed in. +- `index`: Human-friendly ordinal for the instance within its group. +- `id`: Immutable UUID for the instance. Example: diff --git a/content/jammy-migration.md b/content/jammy-migration.md index 5f7802eb1..75564f5a6 100644 --- a/content/jammy-migration.md +++ b/content/jammy-migration.md @@ -1,3 +1,5 @@ +# Migrating Packages to Jammy Jellyfish + Cloud Foundry's upcoming stemcells will be based on Ubuntu's [Jammy Jellyfish](https://wiki.ubuntu.com/Releases) release, which may cause compilation and deployment errors in packages built for earlier stemcells. This document provides guidance on how to address the most common errors that BOSH release authors may encounter. There are three broad categories to address: - GCC 11 — see [below](#gcc-11) @@ -6,7 +8,7 @@ Cloud Foundry's upcoming stemcells will be based on Ubuntu's [Jammy Jellyfish](h Discussion Slack channel is [here](https://app.slack.com/client/T02FL4A1X/C02M2R39Y8Z). -### GCC 11 +## GCC 11 Here's a typical error during compilation phase (`multiple definition of ...`): @@ -31,15 +33,15 @@ Here's another fix from the routing-release's haproxy job's packaging script: make TARGET=linux2628 USE_OPENSSL=1 TARGET_CFLAGS=-fcommon ``` -### OpenSSL 3 +## OpenSSL 3 OpenSSL 3 is a "[is a major release and not fully backwards compatible with the previous release](https://www.openssl.org/blog/blog/2021/09/07/OpenSSL3.Final/)". Jammy includes OpenSSL 3. Although it's unlikely that your package directly uses OpenSSL 3, it's quite possible that one of your package's dependencies do. Bump to a newer version of the dependency which supports OpenSSL 3, for example Ruby 2.7 → 3.1, nginx 1.20.1 → 1.21.6, HAProxy 1.8.13 → 2.5.1. Here are typical failures: -##### When Compiling Ruby 2.7.2 +### When Compiling Ruby 2.7.2 -``` +```shell ossl_pkey_rsa.c:877:58: error: 'RSA_SSLV23_PADDING' undeclared (first use in this function); did you mean 'RSA_NO_PADDING'? ... make[2]: *** [Makefile:313: ossl_pkey_rsa.o] Error 1 @@ -59,9 +61,9 @@ nvim packages/*/spec # update Ruby dependencies to include the new version Note: bumping Ruby versions on any but the most trivial codebases is a significant effort. -##### When Compiling nginx 1.20.1 +### When Compiling nginx 1.20.1 -``` +```shell src/event/ngx_event_openssl.c:5354:5: error: 'ENGINE_free' is deprecated: Since OpenSSL 3.0 [-Werror=deprecated-declarations] 5354 | ENGINE_free(engine); | ^~~~~~~~~~~ @@ -78,23 +80,23 @@ make: *** [Makefile:10: build] Error 2 Bump to nginx 1.21.6 to fix. -##### When Compiling HAProxy 1.8.13: +### When Compiling HAProxy 1.8.13 -``` +```shell make: *** [Makefile:909: src/ssl_sock.o] Error 1 ``` Bump to HAProxy 2.5.1 to fix. -##### When Using `keytool` with OpenJDK 8/11: +### When Using `keytool` with OpenJDK 8/11 -``` +```shell keytool error: java.io.IOException: keystore password was incorrect ``` Use `openssl pkcs12`'s `-legacy` flag when creating the Java keystore. -### Addons (Runtime Configurations) +## Addons (Runtime Configurations) If you restrict your addons to certain stemcells, be sure to include Jammy in your list of stemcells (if you intend your addon to run on Jammy). The following is the updated stemcell list for [cf-deployment](https://github.com/cloudfoundry/cf-deployment)'s manifest: diff --git a/content/job-lifecycle.md b/content/job-lifecycle.md index 6269b5b35..563a23511 100644 --- a/content/job-lifecycle.md +++ b/content/job-lifecycle.md @@ -1,3 +1,5 @@ +# Update Lifecycle + There are several stages that all jobs (and their associated processes) on each VM go through during a deployment process. ## When start is issued {: #start } @@ -30,18 +32,15 @@ There are several stages that all jobs (and their associated processes) on each All lifecycle scripts (pre-start, post-start, post-deploy, pre-stop, drain, post-stop) are spawned at a lower CPU scheduling priority than the BOSH agent itself. This prevents CPU-intensive scripts from starving the agent's event loop, which would otherwise cause the Director to report an agent-unreachable error even though the script is making progress. --- + ## When processes are running {: #running } 1. Monit will automatically restart processes that failed their associated checks - - A common pattern used is a PID check: when no process ID can be found in - any `.pid` file, or the process ID is not alive anymore, then the - process is restarted. - - A usual pitfall arrise when the process ID is not properly written in - the `.pid` file, in which case Monit looses its handle on the actual - process state and things start diverging. Using [bpm](bpm/bpm.md) is an - effective solution to avoid falling in that trap. + - A common pattern used is a PID check: when no process ID can be found in any `.pid` file, or the process ID is not alive anymore, then the process is restarted. + - A usual pitfall arrise when the process ID is not properly written in the `.pid` file, in which case Monit looses its handle on the actual process state and things start diverging. Using [bpm](bpm/bpm.md) is an effective solution to avoid falling in that trap. --- + ## When stop is issued (or before update and subsequent start happens) {: #stop } 1. `monit unmonitor` is called for each process diff --git a/content/job-logs.md b/content/job-logs.md index 3a8b8c3f6..0244de208 100644 --- a/content/job-logs.md +++ b/content/job-logs.md @@ -1,15 +1,18 @@ +# Using Logs + This topic describes different types of logs and how to access them. ## VM logs {: #vm-logs } You can access logs from any VM: -* via [`bosh ssh` command](sysadmin-commands.md#ssh) to SSH into a VM and look at the log files -* via [`bosh logs` command](sysadmin-commands.md#logs) to download logs from the VM +- via [`bosh ssh` command](sysadmin-commands.md#ssh) to SSH into a VM and look at the log files +- via [`bosh logs` command](sysadmin-commands.md#logs) to download logs from the VM The following sections describe different types of logs found on each BOSH managed VM. --- + ### Job logs {: #job-logs } Release jobs on VMs produce logs throughout different lifecycle events. Release authors are strongly encouraged to place release job logs into `/var/vcap/sys/log//*.log`, providing a consistent place for the operator to find them. @@ -27,6 +30,7 @@ See additional information about following job lifecycle events' logs: - [drain script logs](drain.md#logs) --- + ### Errand logs {: #errand-logs } Unlike regular job logs BOSH does not automatically redirect errand logs to `/var/vcap/sys/log/*` directory, though we are planning to do so in future. @@ -45,9 +49,10 @@ bosh run-errand smoke-tests --download-logs --logs-dir ~/workspace/smoke-tests-l ``` !!! note - By default upon errand completion errand VM is deleted, so you cannot access logs saved to disk by the errand. You can use --keep-alive flag when running an errand to keep the VM with its logs. + By default upon errand completion errand VM is deleted, so you cannot access logs saved to disk by the errand. You can use `--keep-alive` flag when running an errand to keep the VM with its logs. --- + ### Monit logs {: #monit-logs } The Agent uses Monit to start, restart, and stop release job processes as specified by the release jobs. Monit detects errors and outputs often useful information to its log. Use `tail` to examine the `monit.log` on a VM: @@ -57,6 +62,7 @@ sudo tail -f -n 200 /var/vcap/monit/monit.log ``` --- + ### Agent logs {: #agent-logs } Agent logs contain configuration and runtime information from the Agent running on a VM. Review these logs if the Director sees VM as unresponsive or the Director fails to contact it during its creation. @@ -71,6 +77,7 @@ sudo tail -f -n 200 /var/vcap/bosh/log/current Agent logs are only accessible to the root user. --- + ### System logs {: #system-logs } System logs contain configuration and runtime information from the Linux kernel and other process running on a VM that are not directly managed by the BOSH Agent. These logs are stored in `/var/log` and are occasionally of interest when debugging OS-level problems, or when determining whether or not a VM is undersized for its workload. `auditd` and `sar` logs are also stored here. @@ -81,6 +88,7 @@ If you're a Linux system administrator, you already know exactly the sorts of th System logs are generally only accessible to the root user. --- + ### Log rotation {: #log-rotation } BOSH rotates release job logs with the [Logrotate][logrotate] log file management utility. Logrotate is configured by the Agent to act on all `.log` files in the `/var/vcap/sys/log/`, `/var/vcap/sys/log/*/`, and `/var/vcap/sys/log/*/*/` directories. @@ -94,11 +102,13 @@ Logs are rotated every 15 minutes (see [agent's `etcLogrotateDTemplate` configur [etcLogrotateDTemplate]: https://github.com/cloudfoundry/bosh-agent/blob/f0b849f/platform/linux_platform.go#L594 --- + ### Syslog configuration {: #syslog-conf } Recommended way to configure syslog forwarding on all or some VMs is to use [`syslog_forwarder` job from `syslog-release` as an addon](addons-common.md#syslog). --- + ## Director task logs {: #director-logs } When you run a [CLI](bosh-cli.md) command, the Director stores all activities for the specific command in a task log. Review these logs when you experience an issue with a command. diff --git a/content/job-templates.md b/content/job-templates.md index c69a6e4d3..f6aa1e649 100644 --- a/content/job-templates.md +++ b/content/job-templates.md @@ -1,3 +1,5 @@ +# Unit Testing Job Templates + ## Unit Testing with `bosh-template` executable {: #unit-testing } `bosh-template` executable in the `bosh-director` gem could be used for unit testing your job templates. Unit testing of job templates becomes even more important once they contain more complex ERB logic that may perform validation or data transformation. diff --git a/content/job-tmpfs.md b/content/job-tmpfs.md deleted file mode 100644 index 934cb2341..000000000 --- a/content/job-tmpfs.md +++ /dev/null @@ -1,61 +0,0 @@ -Job configuration often contains credentials. In some environments we do not -want these values to be written to a physical disk and only ever want them to be -resident in memory. The director has been able to avoid writing credentials to -disk for some time now but they would be written to disk on the deployment VMs. - -The job directory on `tmpfs` feature makes sure that the agent does not write -these values to disk and instead keeps them on an in-memory `tmpfs`. - -## Director Disk - -The director can be configured to load credentials from a secret store, perform -all job template rendering in-memory, and then send the rendered templates over -the message bus directly to the agent. For a complete solution, the -`enable_nats_delivered_templates` feature should be enabled at the same time as -the `tmpfs` agent feature. This feature can be enabled by setting the following -property in your bosh director manifest: - - -``` -- name: bosh - ... - properties: - ... - director: - ... - enable_nats_delivered_templates: true -``` - -If you are using a recent -[bosh-deployment](https://github.com/cloudfoundry/bosh-deployment) then this is -probably already enabled by default. - -## Caveats - -* Due to the job configuration and credentials only being stored in-memory, it - is not possible to successfully reboot VMs which have this feature enabled. - VMs must instead be recreated in order to repopulate them with configuration - and bring them back to a healthy state. - -* The `tmpfs` is 100MB in size by default. We found that this leaves ample room - for most job templates as they are normally small. However, there is a - configuration option to increase the size of the allocated `tmpfs` disk if - necessary. - ---- - -## Job Configuration - -This feature can be enabled by setting the following `env` properties in your -deployment manifest. - -```yaml -instance_groups: -- name: zookeeper - ... - env: - bosh: - job_dir: - tmpfs: true - tmpfs_size: 128m -``` diff --git a/content/jobs.md b/content/jobs.md index e2eece34a..5fa7a4840 100644 --- a/content/jobs.md +++ b/content/jobs.md @@ -1,3 +1,5 @@ +# Release Jobs + Each release job represents a specific chunk of work that the release performs. For example a DHCP release may have a "dhcp-server" job, and a Postgres release may have "postgres" and "periodic-backup" jobs. A release can define one or more jobs. A job typically includes: @@ -11,6 +13,7 @@ A job typically includes: Jobs are typically OS specific (Windows vs Linux); however, structure of a job remains same. --- + ## Spec file (metadata) {: #spec } Spec file defines job metadata. It will be interpreted by the Director when the release is uploaded and when it's deployed. @@ -36,54 +39,61 @@ properties: Schema: -* **name** [String, required]: Name of the job. -* **description** [String, optional]: Describes purpose of the job. -* **templates** [Hash, optional]: [Template files](#templates) found in the +- **name** [String, required]: Name of the job. +- **description** [String, optional]: Describes purpose of the job. +- **templates** [Hash, optional]: [Template files](#templates) found in the `templates` directory of the job (keys of the Hash) and their final destinations (values of the Hash), relative to the job directory on the deployed VMs. - * **<key>** [String, required]: the relative path and filename of the + + - **<key>** [String, required]: the relative path and filename of the ERB template provided by the job in the release, relative to the `templates` sub-directory. No need for any `.erb` suffix, all templates are treated as ERB templates whatever their name is. - * **<value>** [String, required]: the relative path and filename of the + - **<value>** [String, required]: the relative path and filename of the rendered file, relative to the job directory (i.e. `/var/vcap/jobs//`) on the managed Bosh instances (a.k.a. the “deployed VMs”). By convention, executable files should be placed into `bin/` directory so that the Agent can mark them as executables, and configuration files should be placed into `config/` directory. -* **packages** [Array, optional]: Package dependencies required by the job at runtime. -* **consumes** [Array, optional]: Links that are consumed by the job for + +- **packages** [Array, optional]: Package dependencies required by the job at runtime. +- **consumes** [Array, optional]: Links that are consumed by the job for rendering ERB templates. - * **name** [String, required]: Name of the link to find. - * **type** [String, required]: Type of the link to be found. This is an + + - **name** [String, required]: Name of the link to find. + - **type** [String, required]: Type of the link to be found. This is an arbitrary naming. Usual and conventional types are `address` when the link goal is to expose a Bosh DNS name that allows accessing the instances of the group. Usually typed by technology, like `mysql`, `postgres`, `cassandra`, etc. Anything that makes sense is relevant and matters. - * **optional** [Boolean, optional]: Whether finding an matching link is + - **optional** [Boolean, optional]: Whether finding an matching link is optional (when `true`) or mandatory (when `false`. Default is `false`, so optional links must be explicitly declared as such. -* **provides** [Array, optional]: Links that are exposed to other jobs for + +- **provides** [Array, optional]: Links that are exposed to other jobs for rendering their ERB templates. - * **name** [String, required]: Name of the exposed link. - * **type** [String, required]: Type of the exposed link. - * **properties** [Array, optional]: List of property keys in dot notation + - **name** [String, required]: Name of the exposed link. + - **type** [String, required]: Type of the exposed link. + - **properties** [Array, optional]: List of property keys in dot notation (same as **properties.<name>** below) -* **properties** [Hash, optional]: Configuration options supported by the job. - * **<name>** [String, required]: Property key in dot notation. Typical + +- **properties** [Hash, optional]: Configuration options supported by the job. + + - **<name>** [String, required]: Property key in dot notation. Typical properties include account names, passwords, shared secrets, hostnames, IP addresses, port numbers, and descriptions. - * **description** [String, required]: Describes purpose of the + + - **description** [String, required]: Describes purpose of the property. This is not used by the Director, but is displayed in job configuration details provided by the [release index](/releases). - * **type** [String, optional]: The type of the property. This is only + - **type** [String, optional]: The type of the property. This is only a convention for release authors to provide a type when they estimate it useful. Example: `type: certificate`. - * **example** [Any, optional]: Example value, to be displayed in the + - **example** [Any, optional]: Example value, to be displayed in the [release index](/releases). Default is `nil`. - * **default** [Any, optional]: The default value for the property. + - **default** [Any, optional]: The default value for the property. Default is `nil`. !!! Note @@ -97,6 +107,7 @@ Schema: [concourse_web_spec]: https://github.com/concourse/concourse-bosh-release/blob/8d2cfa0/jobs/web/spec#L68-L71 --- + ## Templates (ERB configuration files) {: #templates } Release authors can define zero or more templates for each job, but typically @@ -404,36 +415,36 @@ Remember that the job targeted through alink can live in a different instance group of a different deployment. - Structural info - - `link(name).deployment_name`: Deployment name of the linked job. - - `link(name).instance_group`: Instance group name of the linked job. - - `link(name).group_name`: A concatenation of the link name and link type, + - `link(name).deployment_name`: Deployment name of the linked job. + - `link(name).instance_group`: Instance group name of the linked job. + - `link(name).group_name`: A concatenation of the link name and link type, separated by a dash `-`, i.e. `-`. - - `link(name).instances`: An array of details for each instance of the group. - - `link(name).instances[].az`: the availability zone hat the instance is + - `link(name).instances`: An array of details for each instance of the group. + - `link(name).instances[].az`: the availability zone hat the instance is placed into - - `link(name).instances[].name`: instance group name. Alias for + - `link(name).instances[].name`: instance group name. Alias for `link().instance_group`. - - `link(name).instances[].id`: instance immutable UUID - - `link(name).instances[].index`: human-friendly instance ordinal - - `link(name).instances[].bootstrap`: whether the instance is the first of + - `link(name).instances[].id`: instance immutable UUID + - `link(name).instances[].index`: human-friendly instance ordinal + - `link(name).instances[].bootstrap`: whether the instance is the first of its group - Networking setup - - `link(name).default_network`: default network for the instance group. - - `link(name).networks`: list of all networks for the instance group. **TO BE TESTED** - - `link(name).address`: an address for the instance group, using the `q-s0` + - `link(name).default_network`: default network for the instance group. + - `link(name).networks`: list of all networks for the instance group. **TO BE TESTED** + - `link(name).address`: an address for the instance group, using the `q-s0` prefix, indicating the `smart` health filter. See [Native DNS Support](dns.md) for more details. - - `link(name).domain`: the root top-level domain name suffix. Defaults to `bosh`. - - `link(name).use_link_dns_names`: applicable config for the link. **TO BE TESTED** - - `link(name).use_short_dns_addresses`: applicable config for the link. **TO BE TESTED** - - `link(name).instances[].address`: the instance address, that can be an + - `link(name).domain`: the root top-level domain name suffix. Defaults to `bosh`. + - `link(name).use_link_dns_names`: applicable config for the link. **TO BE TESTED** + - `link(name).use_short_dns_addresses`: applicable config for the link. **TO BE TESTED** + - `link(name).instances[].address`: the instance address, that can be an IPv4, an IPv6 address or a DNS record, depending on the Director's configuration, but is usually a DNS name, ending with the suffix indicated in the `link().domain` property. - - `link(name).instances[].addresses`: several addresses including aliases? **TO BE TESTED** - - `link(name).instances[].dns_addresses`: same as above, but preferring DNS entry + - `link(name).instances[].addresses`: several addresses including aliases? **TO BE TESTED** + - `link(name).instances[].dns_addresses`: same as above, but preferring DNS entry - Configuration - - `link(name).properties`: The job properties that are exposed by the link. + - `link(name).properties`: The job properties that are exposed by the link. ##### Deprecated properties accessors diff --git a/content/jumpbox.md b/content/jumpbox.md index e93d7f356..f9cc1d609 100644 --- a/content/jumpbox.md +++ b/content/jumpbox.md @@ -1,3 +1,5 @@ +# Enabling SSH Access + It's recommended: - to maintain a separate jumpbox VM for your environment diff --git a/content/links-api.md b/content/links-api.md index 689a76d38..60bebfebf 100644 --- a/content/links-api.md +++ b/content/links-api.md @@ -8,12 +8,14 @@ See [Links](links.md) for an introduction on links, link providers, and link consumers. This API is hosted on the director. ## Providers + ### `GET /link_providers`: List Providers Obtain an array of providers created in a deployment. #### Request Query Params -* **deployment**: [String] Deployment name. + +- **deployment**: [String] Deployment name. ```shell bosh curl /link_providers?deployment=zookeeper @@ -67,7 +69,8 @@ Should result in: Obtain an array of consumers created for a deployment. #### Request Query Params -* **deployment**: [String] Deployment name. + +- **deployment**: [String] Deployment name. ```shell bosh curl /link_consumers?deployment=zookeeper @@ -122,7 +125,8 @@ Should result in: Obtain an array of links created for a deployment. #### Request Query Params -* **deployment**: [String] Deployment name. + +- **deployment**: [String] Deployment name. ```shell bosh curl /links?deployment=zookeeper @@ -157,15 +161,17 @@ Create an external link with a user-defined consumer. Uses an existing provider. The UAA client creating the link must have a **full admin** or **team admin** scope. See [Director Users and Permissions](https://bosh.io/docs/director-users-uaa-perms/) for details. #### Request Headers -* `Content-Type: application/json` + +- `Content-Type: application/json` #### Request Schema -* **link_provider_id**: [String] The id corresponding to the existing link provider. -* **link_consumer**: - * **owner_object**: - * **name**: [String] The name for the new consumer. - * **type**: [String] Type is always "external". -* **network**: [String] Name of a network used by the provider (optional). See [custom network linking](links.md#custom-network). + +- **link_provider_id**: [String] The id corresponding to the existing link provider. +- **link_consumer**: + - **owner_object**: + - **name**: [String] The name for the new consumer. + - **type**: [String] Type is always "external". +- **network**: [String] Name of a network used by the provider (optional). See [custom network linking](links.md#custom-network). ```shell bosh curl -X POST -H 'Content-Type: application/json' --body <(echo '{ @@ -180,10 +186,11 @@ bosh curl -X POST -H 'Content-Type: application/json' --body <(echo '{ ``` #### Response Schema -* **created_at**: [Date] Timestamp. -* **link_provider_id**: [String] ID of the provider used. -* **link_consumer_id**: [String] ID of the new consumer created for this link. -* **name**: [String] The name of the link, set from the provider. + +- **created_at**: [Date] Timestamp. +- **link_provider_id**: [String] ID of the provider used. +- **link_consumer_id**: [String] ID of the new consumer created for this link. +- **name**: [String] The name of the link, set from the provider. ```json { @@ -204,16 +211,17 @@ Delete links created with this API. #### Request -* **link-id**: [String] ID of link to delete. +- **link-id**: [String] ID of link to delete. ```shell bosh curl -X DELETE /links/3 ``` #### Response -* **HTTP 204**: Deleted successfully. -* **HTTP 404**: Link not found. -* **HTTP 400**: Bad request. Only links with type `external` can be deleted. + +- **HTTP 204**: Deleted successfully. +- **HTTP 404**: Link not found. +- **HTTP 400**: Bad request. Only links with type `external` can be deleted. ## Link Address @@ -222,16 +230,18 @@ bosh curl -X DELETE /links/3 Obtain the DNS address for a singular link. This is equivalent to using `link("my-link").address` in jobs templates. #### Request Params -* **link_id**: [String] The link ID. -* **azs[]**: [String] Name of the AZ to filter by (optional). This parameter should be provided multiple times when specifying multiple availability zones; see example below. -* **status**: [String] Filter by health status. One of: healthy, unhealthy, all, default (optional). + +- **link_id**: [String] The link ID. +- **azs[]**: [String] Name of the AZ to filter by (optional). This parameter should be provided multiple times when specifying multiple availability zones; see example below. +- **status**: [String] Filter by health status. One of: healthy, unhealthy, all, default (optional). ```shell bosh curl '/link_address?link_id=3&azs[]=z1' ``` #### Response Body -* **address**: [String] DNS address for the link. + +- **address**: [String] DNS address for the link. ```json { @@ -240,6 +250,7 @@ bosh curl '/link_address?link_id=3&azs[]=z1' ``` #### Specifying Multiple AZs in the Same Request + The `azs[]` parameter should be provided multiple times when specifying multiple AZs in the query request. For example, to filter by availability zones **z1**, **z2**, and **z3**, the request will look like: ```shell diff --git a/content/links-common-types.md b/content/links-common-types.md index cf0ae4d22..83f58d0fa 100644 --- a/content/links-common-types.md +++ b/content/links-common-types.md @@ -1,23 +1,27 @@ +# Links Common Types + Common suggested link types that release authors should conform to if type is used. --- + ## `database` type {: #database } `link('...').address` returns database address. -* **adapter** [String]: Name of the adapter. -* **port** [Integer]: Port to connect on. -* **tls** [Hash]: See [TLS properties](props-common.md#tls) -* **username** [String]: Database username to use. Example: `u387455`. -* **password** [String]: Database password to use. -* **database** [String]: Database name. Example: `app-server` -* **options** [Hash, optional]: Opaque set of options for the adapter. Example: `{max_connections: 32, pool_timeout: 10}`. +- **adapter** [String]: Name of the adapter. +- **port** [Integer]: Port to connect on. +- **tls** [Hash]: See [TLS properties](props-common.md#tls) +- **username** [String]: Database username to use. Example: `u387455`. +- **password** [String]: Database password to use. +- **database** [String]: Database name. Example: `app-server` +- **options** [Hash, optional]: Opaque set of options for the adapter. Example: `{max_connections: 32, pool_timeout: 10}`. --- + ## `blobstore` type {: #blobstore } `link('...').address` returns blobstore address. -* **adapter** [String]: Name of the adapter (s3, gcp, etc.). -* **tls** [Hash]: See [TLS properties](props-common.md#tls) -* **options** [Hash, optional]: Opaque set of options. Example: `{aws_access_key: ..., secret_access_key: ...}`. +- **adapter** [String]: Name of the adapter (s3, gcp, etc.). +- **tls** [Hash]: See [TLS properties](props-common.md#tls) +- **options** [Hash, optional]: Opaque set of options. Example: `{aws_access_key: ..., secret_access_key: ...}`. diff --git a/content/links-manual.md b/content/links-manual.md index 7c8fd7e76..ff296be22 100644 --- a/content/links-manual.md +++ b/content/links-manual.md @@ -1,3 +1,5 @@ +# Manually Configuring Links + (See [Links](links.md) and [Link properties](links-properties.md) for an introduction.) !!! note diff --git a/content/links-properties.md b/content/links-properties.md index 8f3591476..7b8547b3f 100644 --- a/content/links-properties.md +++ b/content/links-properties.md @@ -1,3 +1,5 @@ +# Link Properties + (See [Links](links.md) for an introduction.) !!! note @@ -108,5 +110,6 @@ instance_groups: properties: password: some-password ``` + !!! note - There is a [bosh issue](https://github.com/cloudfoundry/bosh/issues/2250) with sharing of hash properties via Credhub. \ No newline at end of file + There is a [bosh issue](https://github.com/cloudfoundry/bosh/issues/2250) with sharing of hash properties via Credhub. diff --git a/content/links.md b/content/links.md index 6a5765c29..72145ae02 100644 --- a/content/links.md +++ b/content/links.md @@ -1,3 +1,5 @@ +# Links + !!! note This feature is available with bosh-release v255.5+. @@ -6,6 +8,7 @@ Previously, if network communication was required between jobs, release authors Links provide a solution to the above problem by making the Director responsible for the IP management. Release authors get a consistent way of retrieving networking (and topology) configuration, and operators have a way to consistently connect components. --- + ## Overview {: #overview } First, we provide an overview of the capabilities and logistics of links through a simple example. Here, we have two jobs: an application job and a database job. The database provides its connection information through a link, which the application consumes. @@ -98,6 +101,7 @@ instance_groups: ``` --- + ## Release Definitions {: #definition } Instead of defining properties for every instance group, a job can declare links. (The job either 'consumes' a link provided by another job, or it can 'provide' itself so that any jobs, [including itself](#self) can 'consume' it). @@ -108,7 +112,8 @@ For example, here is how a `web` job which receives HTTP traffic and talks to at Note that when the `web` job is 'consuming' db links, the name of the link does not have to match the name of the provided db link (i.e. postgres has a link called `conn` while the `web` job consumes `primary_db` and/or `secondary_db`). The mapping between the provided link named `conn` and the consumed link named `primary_db` is done in the [deployment manifest file](#deployment). -#### `web` Release Spec {: #web-release-spec} +### `web` Release Spec {: #web-release-spec} + ```yaml name: web @@ -132,7 +137,8 @@ Note that the `secondary_db` link has been marked as optional, to indicate that Here is an example Postgres job that provides a `conn` link of type `db`. -#### `postgres` Release Spec {: #postgres-release-spec} +### `postgres` Release Spec {: #postgres-release-spec} + ```yaml name: postgres @@ -145,7 +151,7 @@ provides: properties: {...} ``` -### Template Accessors {: #templates } +## Template Accessors {: #templates } Once a release is configured to consume links, the `link` template accessor allows access to link information such as instance names, AZs, IDs, network addresses, etc. @@ -188,25 +194,26 @@ JSON.dump(result) Available `link` object methods: -* **address** [String]: Returns single DNS address representing link provider. Using single address is typically a more common way to reference a link provider instead of accessing individual instance addresses (for example, when connecting to a database). Example: `link("...").address`. - * **azs** [Array of strings, optional]: Argument to filter instance addresses by AZ. Logical OR will be used between AZs when multiple AZs are specified. Example: `link("...").address(azs: [spec.az])`. Default: all instances are returned without AZ filtering. -* **p** [Anything]: Returns property value specified in a link. Works in the same way as regular `p` accessor. -* **instances** [Array of instances]: Returns list of instances included by this provider. Could be an empty array. See methods available on each instance below. +- **address** [String]: Returns single DNS address representing link provider. Using single address is typically a more common way to reference a link provider instead of accessing individual instance addresses (for example, when connecting to a database). Example: `link("...").address`. +- **azs** [Array of strings, optional]: Argument to filter instance addresses by AZ. Logical OR will be used between AZs when multiple AZs are specified. Example: `link("...").address(azs: [spec.az])`. Default: all instances are returned without AZ filtering. +- **p** [Anything]: Returns property value specified in a link. Works in the same way as regular `p` accessor. +- **instances** [Array of instances]: Returns list of instances included by this provider. Could be an empty array. See methods available on each instance below. Available `instance` object methods: -* **name** [String, non-empty]: Instance name as configured in the deployment manifest. -* **id** [String, non-empty]: Unique ID. -* **index** [Integer, non-empty]: Unique numeric index. May have gaps. -* **az** [String or null, non-empty]: AZ associated with the instance. -* **address** [String, non-empty]: IPv4, IPv6 or DNS address. See [Native DNS Support](dns.md#links) for more details. -* **bootstrap** [Boolean]: Whether or not this instance is a bootstrap instance. +- **name** [String, non-empty]: Instance name as configured in the deployment manifest. +- **id** [String, non-empty]: Unique ID. +- **index** [Integer, non-empty]: Unique numeric index. May have gaps. +- **az** [String or null, non-empty]: AZ associated with the instance. +- **address** [String, non-empty]: IPv4, IPv6 or DNS address. See [Native DNS Support](dns.md#links) for more details. +- **bootstrap** [Boolean]: Whether or not this instance is a bootstrap instance. -### Properties {: #properties } +## Properties {: #properties } See [link properties](links-properties.md) for including additional link information. --- + ## Deployment Configuration {: #deployment } Given the `web` and `postgres` job examples above, one can configure a deployment that connects a web app to the database. The following example demonstrates linking defined explicitly in the manifest by saying which jobs provide and consume a link `db_conn`. @@ -238,6 +245,7 @@ Implicit linking is not supported between deployments. Providers that are specified as `nil` will not match any consumer. Deployment Manifest: + ```yaml instance_groups: - name: app_ig @@ -278,22 +286,22 @@ Common use cases: There are two use cases that require the use of *explicit* linking. -* To distinguish between multiple providers of the same `type` in a deployment. -* To consume a link provided by a different deployment. - +- To distinguish between multiple providers of the same `type` in a deployment. +- To consume a link provided by a different deployment. #### Consumers Explicitly defined consumers can have the following optional properties: -* **from** [String]: Overrides the name of the provider to consume. This should match the name defined in the provider's release spec or the name defined by provider's `as` property in the manifest. -* **deployment** [String]: The name of the deployment to consume from. If the deployment provided does not exist, the consumer will fail with an error. The default value for this property is the name of the current deployment, which means that consumers are expected to be in the same deployment as the provider. -* **network** [String]: Network to be used by the consumer. This must match the name of one of the networks defined by the provider. The default value is the provider's default network. See [custom network linking](#custom-network). -* **ip_addresses** [Boolean]: Instructs the director to use ip addresses instead of DNS names. This property is ignored in the case of dynamic networks, which always use DNS addresses. Defaults to *false*. See [dns](dns.md#links) for more details. +- **from** [String]: Overrides the name of the provider to consume. This should match the name defined in the provider's release spec or the name defined by provider's `as` property in the manifest. +- **deployment** [String]: The name of the deployment to consume from. If the deployment provided does not exist, the consumer will fail with an error. The default value for this property is the name of the current deployment, which means that consumers are expected to be in the same deployment as the provider. +- **network** [String]: Network to be used by the consumer. This must match the name of one of the networks defined by the provider. The default value is the provider's default network. See [custom network linking](#custom-network). +- **ip_addresses** [Boolean]: Instructs the director to use ip addresses instead of DNS names. This property is ignored in the case of dynamic networks, which always use DNS addresses. Defaults to *false*. See [dns](dns.md#links) for more details. Optional consumers may be specified as `nil` in the deployment manifest to block consumption of any providers. Deployment Manifest: + ```yaml instance_groups: - name: web_ig @@ -307,7 +315,8 @@ instance_groups: Explicitly specified providers in the deployment manifest can have the following optional properties: -* **as** [String]: Overrides the name of the provider defined in the release spec. Example: +- **as** [String]: Overrides the name of the provider defined in the release spec. Example: + ```yaml instance_groups: - name: my_instance_group @@ -316,7 +325,8 @@ instance_groups: provides: conn: {as: new_name} ``` -* **shared** [Boolean]: *Default is false* Sets whether this provider is consumable from another deployment. See [cross deployment links](#cross-deployment). + +- **shared** [Boolean]: *Default is false* Sets whether this provider is consumable from another deployment. See [cross deployment links](#cross-deployment). This applies whether the consumer is explicit or implicit. @@ -325,6 +335,7 @@ This applies whether the consumer is explicit or implicit. Providers that are specified as `nil` in the deployment manifest cannot be consumed. Deployment Manifest: + ```yaml instance_group: - name: db_ig @@ -336,7 +347,6 @@ instance_group: A common use case for this is when a provider is optional and the current deployment will possibly use an alternative provider. - ### Self linking {: #self } A job can consume a link that it provides. This could be used to determine a job's own peers. @@ -490,37 +500,39 @@ instance_groups: - port - url ``` -___ + +--- ## Avoiding Link Conflicts When writing a manifest that contains multiple jobs that provide a link, deployment will sometimes fail because of conflicts stemming from the links' names and types. Here are two typical errors: -``` +```shell Failed to resolve link 'login' with alias 'provider_login' and type 'usernamepassword' from job 'consumer_job' in instance group 'consumer_ig'. Details below: - No link providers found ``` -``` +```shell - Failed to resolve link 'provider' with alias 'alias1' and type 'provider' from job 'consumer' in instance group 'first_consumer'. Multiple link providers found: - Link provider 'provider' with alias 'alias1' from job 'provider' in instance group 'first_provider' in deployment 'simple' - Link provider 'provider' with alias 'alias1' from job 'provider' in instance group 'second_provider' in deployment 'simple' ``` -How to prevent such errors depends on how the links are consumed. +How to prevent such errors depends on how the links are consumed. ### No Consumers -As long as they are not consumed, multiple providers in a deployment manifest will never generate errors during deployment even if the names or types of the individual job providers are the same as each other. -| Provider Name | Provider Type | Allowed | Example (below) -|---------------|---------------|---------|-------------- -| Same | Same | True | `database` of type `db` in both `db_ig` and `backup_db_ig` -| Same | Different | True | `peers` of type `db_peers` in `db_ig` and `peers` of type `legacy_db_peers` in `backup_db_ig` -| Different | Same | True | `peers` in `db_ig` and `backup_peers` both of type `db` in `backup_db_ig` -| Different | Different | True | `database` of type `db` and `backup_peers` of type `db_peers` +As long as they are not consumed, multiple providers in a deployment manifest will never generate errors during deployment even if the names or types of the individual job providers are the same as each other. +| Provider Name | Provider Type | Allowed | Example (below) | +|---------------|---------------|---------|-----------------------------------------------------------------------------------------------| +| Same | Same | True | `database` of type `db` in both `db_ig` and `backup_db_ig` | +| Same | Different | True | `peers` of type `db_peers` in `db_ig` and `peers` of type `legacy_db_peers` in `backup_db_ig` | +| Different | Same | True | `peers` in `db_ig` and `backup_peers` both of type `db` in `backup_db_ig` | +| Different | Different | True | `database` of type `db` and `backup_peers` of type `db_peers` | Example manifest for table above. + ```yaml instance_groups: - name: db_ig @@ -577,22 +589,21 @@ instance_groups: There are two possible ways to fix this: -* Add the `provides` section of the providing job in the release manifest to rename the link using the `as` property. -* Introduce a second release to provide a backup job with a different link name or type. +- Add the `provides` section of the providing job in the release manifest to rename the link using the `as` property. +- Introduce a second release to provide a backup job with a different link name or type. -___ +--- ## Links FAQ -Q: What characters are valid for link names?
+Q: What characters are valid for link names? A: All Unicode characters can be used in names. However a name cannot begin with a colon (`:`). -Q: When are cross-deployment links resolved?
+Q: When are cross-deployment links resolved? A: They are only resolved during a deployment of the consumer. The provider is ready for consumption after a successful deploy of the provider deployment. -Q: If a cross-deployment provider is deleted what happens to the consumer?
+Q: If a cross-deployment provider is deleted what happens to the consumer? A: Consumers will continue to have access to the link until the consumer deployment is redeployed. Consumer VMs can be recreated without losing the links' values. On redeployment, links will no longer resolve if the provider is no longer available. -Q: Are releases the only entities that can provide and consume links?
+Q: Are releases the only entities that can provide and consume links? A: No, there are other entities that can provide links, such as [manual links](links-manual.md), and [external links](links-api.md). There are also [custom link providers](links.md#custom-provider-definitions) which variables can use to [consume some DNS values](dns.md#dns-variables-integration). - diff --git a/content/locking-compiled-releases.md b/content/locking-compiled-releases.md index bc9801b16..5d56c27ea 100644 --- a/content/locking-compiled-releases.md +++ b/content/locking-compiled-releases.md @@ -1,6 +1,9 @@ +# Locking Compiled Releases + !!! note Available as of BOSH Director version [v268.7.0](https://github.com/cloudfoundry/bosh/releases/tag/v268.7.0) -# Motivation + +## Motivation When deploying instance groups that use compiled releases, if the release is not compiled against the exact stemcell used by the instance group, @@ -28,7 +31,7 @@ Bosh will use the new release compiled against the newer stemcell. To lock a compiled release and reduce unexpected VM updates, use `exported_from` in the [releases](deployment-manifest.md#releases) block of your manifest. -# Usage +## Usage To specify which compiled release to use in a deployment, add `exported_from` to the release: @@ -53,7 +56,7 @@ To create a compiled release, follow the instructions on the [compiled releases `exported_from` is an array to support future use cases, such as matching multiple stemcells. Currently, only the first entry in `exported_from` is used. -# Caveats +## Caveats - The `exported_from` field is designed for compiled releases. If using a source release with `exported_from`, diff --git a/content/managed-networks.md b/content/managed-networks.md index edef8ff49..b0d0a0bf5 100644 --- a/content/managed-networks.md +++ b/content/managed-networks.md @@ -1,3 +1,5 @@ +# Managed Networks + !!! note Available as of BOSH Director version 268.1.0 @@ -9,7 +11,7 @@ When a platform operator defines a logical network at the `cloud config` manifes To enable the managed network lifecycle, the Operator needs to enable it in `cloud config`, making sure that `managed` is set to `true` under the network definition: -``` +```yaml networks: name: my-network type: manual @@ -23,7 +25,8 @@ networks: ``` The current status of `network_lifecycle` can be checked on BOSH environment info: -``` + +```shell $ bosh environment Using environment '30.0.0.1' as client 'admin' @@ -44,9 +47,8 @@ Succeeded ![image](images/managed_network_lifecycle.png) - ## Implementation CPI authors looking to provide managed networks functionality should refer to [create_network](cpi-api-v2-method/create-network.md) and [delete_network](cpi-api-v2-method/delete-network.md) methods documentation. -[vSphere CPI](vsphere-cpi.md#networks) currently provides a concrete implementation of managed networks. \ No newline at end of file +[vSphere CPI](vsphere-cpi.md#networks) currently provides a concrete implementation of managed networks. diff --git a/content/managing-releases.md b/content/managing-releases.md index d295699cc..3ac4eb6f0 100644 --- a/content/managing-releases.md +++ b/content/managing-releases.md @@ -1,6 +1,9 @@ +# Managing Releases + (See [What is a Release?](release.md) and [Uploading releases](uploading-releases.md) for an introduction.) --- + ## Jobs and Packages {: #jobs-and-packages } Each job and package is uniquely identified by its name and fingerprint. A fingerprint is calculated based on the contents of all associated files, their permissions. A release captures set of job and package versions that depend on each other and gives it a name and a version. The CLI also records additional metadata when creating releases such as Git SHA. @@ -55,6 +58,7 @@ Job and package names and fingerprints are used throughout the system to identif - to determine if instances need to be updated during a [deployment procedure](deploying-step-by-step.md) --- + ## Version uniqueness {: #uniqueness } There exists an implicit trust between a release author and an operator that different release (a different set of jobs and packages) will not be published under the same version. The Director will reject new set of jobs and packages during the upload if it does not match with already uploaded contents. @@ -62,6 +66,7 @@ There exists an implicit trust between a release author and an operator that dif Sometimes however two different releases (e.g. 1.0.1 and 1.0.3) may end up with exactly same set of jobs and packages. In fact running [`bosh create-release` command](cli-v2.md#create-release) a few times in a row will produce new releases with only version being different (e.g. 1.0.1+dev.0 and 1.0.1+dev.1). Operators should be aware that even if the deployment procedure shows release version changing there is a chance that no instances will be updated. This initially may be a surprising behaviour; however, given that the Director correctly determines that there are no changes to apply to the instances, there is really nothing to do. After the deploy is finished, [`bosh deployments` command](cli-v2.md#deployments) will of course state that new release is used. --- + ## Inspecting uploaded releases {: #inspect } Once release is uploaded to the Director, it can be inspected via [`bosh inspect-release` command](cli-v2.md#inspect-release). It will show names and fingerprints of jobs and packages. It will also show job details such as consumed and provided links. @@ -70,7 +75,7 @@ Once release is uploaded to the Director, it can be inspected via [`bosh inspect bosh -e vbox inspect-release zookeeper/0.0.5 ``` -```text +```shell Using environment '192.168.56.6' as '?' Job Blobstore ID Digest Links Consumed Links Provided @@ -101,11 +106,13 @@ Succeeded For debugging command also shows blobstore information (ID and SHA1) for each job and package. The Director uses blobstore references when deploying jobs and compiling packages. --- + ## Fixing corrupted releases (experimental) {: #fix } Assuming that somehow the Director blobstore loses referenced asset (job, source or compiled package), it's possible to fix the corrupted asset. [`bosh upload-release` command](cli-v2.md#upload-release) provides a `--fix` flag which allows to reupload same release contents into the Director. --- + ## Cleaning up uploaded releases {: #clean-up } Over time the Director accumulates releases, hence it uses more blobstore space. diff --git a/content/managing-stemcells.md b/content/managing-stemcells.md index 88d407cb9..5caf290b5 100644 --- a/content/managing-stemcells.md +++ b/content/managing-stemcells.md @@ -1,11 +1,15 @@ +# Managing Stemcells + (See [What is a Stemcell?](stemcell.md) and [Uploading stemcells](uploading-stemcells.md) for an introduction.) --- + ## Stemcell versioning {: #versioning } Each stemcell is uniquely identified by its name and version. Currently stemcell versions are in MAJOR.MINOR format. Major version is incremented when new features are added to stemcells (or any components that stemcells typically include such as BOSH Agent). Minor versions are incremented if certain security fixes and/or features are backported on top of existing stemcell line. We recommend to continuously bump to the latest major stemcell version to receive latest updates. --- + ## Overview {: #overview } You can identify stemcell version from inside the VM via following files: @@ -19,11 +23,13 @@ You can identify stemcell version from inside the VM via following files: See [Stemcell Building](build-stemcell.md#tarball-structure) to find stemcell archive structure. --- + ## Fixing corrupted stemcells {: #fix } Occasionally stemcells are deleted from the IaaS outside of the Director. For example your vSphere administrator decided to clean up your vSphere VMs folder. The Director of course will continue to reference deleted IaaS asset and CPI will eventually raise an error when trying to create new VM. [`bosh upload-stemcell` command](cli-v2.md#upload-stemcell) provides a `--fix` flag which allows to reupload stemcell with the same name and version into the Director fixing this problem. --- + ## Cleaning up uploaded stemcells {: #clean-up } Over time the Director accumulates stemcells. Stemcells could be deleted manually via [`bosh delete-stemcell` command](cli-v2.md#delete-stemcell) or be cleaned up via [`bosh cleanup` command](cli-v2.md#clean-up). diff --git a/content/manifest-v2.md b/content/manifest-v2.md index 22b5cc24d..442f581a6 100644 --- a/content/manifest-v2.md +++ b/content/manifest-v2.md @@ -1,3 +1,5 @@ +# Deployment Manifest v2 + !!! note This feature is available with bosh-release v255.4+. @@ -8,18 +10,19 @@ The deployment manifest is a YAML file that defines the components and propertie Assuming that you are using [cloud config](cloud-config.md), your deployment manifest is expected to have: -* [Deployment Identification](#deployment): A name for the deployment and the UUID of the Director managing the deployment -* [Features Block](#features): Opts into Director features to be used in this deployment -* [Releases Block](#releases): Name and version of each release in a deployment -* [Stemcells Block](#stemcells): Name and version of each stemcell in a deployment -* [Update Block](#update): Defines how BOSH updates instances during deployment -* [Instance Groups Block](#instance-groups): Configuration and resource information for instance groups -* [Addons](#addons): Configures deployment specific addons -* [Properties Block](#properties): Describes global properties and generalized configuration information -* [Variables Block](#variables): Variables configuration -* [Tags Block](#tags): Sets additional tags for the deployment +- [Deployment Identification](#deployment): A name for the deployment and the UUID of the Director managing the deployment +- [Features Block](#features): Opts into Director features to be used in this deployment +- [Releases Block](#releases): Name and version of each release in a deployment +- [Stemcells Block](#stemcells): Name and version of each stemcell in a deployment +- [Update Block](#update): Defines how BOSH updates instances during deployment +- [Instance Groups Block](#instance-groups): Configuration and resource information for instance groups +- [Addons](#addons): Configures deployment specific addons +- [Properties Block](#properties): Describes global properties and generalized configuration information +- [Variables Block](#variables): Variables configuration +- [Tags Block](#tags): Sets additional tags for the deployment --- + ## Deployment Identification {: #deployment } **name** [String, required]: The name of the deployment. A single Director can manage multiple deployments and distinguishes them by name. @@ -33,6 +36,7 @@ name: my-redis ``` --- + ## Features Block {: #features } **features** [Hash, options]: Specifies Director features that should be used within this deployment. @@ -44,11 +48,11 @@ name: my-redis that you use the [`update_mode` configuration](#variables) on the variable that you wish to converge. -* **converge\_variables** [Boolean, optional]: Enable variables to be regenerated by the backend config server (e.g. [CredHub](https://docs.cloudfoundry.org/api/credhub/version/2.9/#_overwriting_credential_values)) when the variable `options` change. Default `false`. Available in bosh-release 267+. -* **randomize\_az\_placement** [Boolean, optional]: Randomizes AZs for left over instances that cannot be distributed equally between AZs. For example, given an instance group with 5 instances and only 3 AZs, 1 remaining instance will be placed in randomly chosen AZ out of specified 3 AZs. Available in bosh-release 264+. -* **use\_dns\_addresses** [Boolean, optional]: Enables or disables returning of DNS addresses in links. Defaults to global Director `use_dns_addresses` configuration. -* **use\_tmpfs\_config** [Boolean, optional]: Mounts all directories which contain [rendered job templates on tmpfs](creds-tmpfs.md) by default on every instance group. Default `false`. -* **use_short_dns_addresses** [Boolean, optional]: Uses a shorter DNS query format to reduce length of DNS name, for use in certificate common names. Default: `false`. See [DNS and variables integration](https://bosh.io/docs/dns/#dns-variables-integration) for more. +- **converge\_variables** [Boolean, optional]: Enable variables to be regenerated by the backend config server (e.g. [CredHub](https://docs.cloudfoundry.org/api/credhub/version/2.9/#_overwriting_credential_values)) when the variable `options` change. Default `false`. Available in bosh-release 267+. +- **randomize\_az\_placement** [Boolean, optional]: Randomizes AZs for left over instances that cannot be distributed equally between AZs. For example, given an instance group with 5 instances and only 3 AZs, 1 remaining instance will be placed in randomly chosen AZ out of specified 3 AZs. Available in bosh-release 264+. +- **use\_dns\_addresses** [Boolean, optional]: Enables or disables returning of DNS addresses in links. Defaults to global Director `use_dns_addresses` configuration. +- **use\_tmpfs\_config** [Boolean, optional]: Mounts all directories which contain [rendered job templates on tmpfs](creds-tmpfs.md) by default on every instance group. Default `false`. +- **use_short_dns_addresses** [Boolean, optional]: Uses a shorter DNS query format to reduce length of DNS name, for use in certificate common names. Default: `false`. See [DNS and variables integration](https://bosh.io/docs/dns/#dns-variables-integration) for more. Example: @@ -58,20 +62,21 @@ features: ``` --- + ## Releases Block {: #releases } **releases** [Array, required]: The name and version of each release in the deployment. -* **name** [String, required]: Name of a release used in the deployment. -* **version** [String, required]: The version of the release to use. Version can be `latest` or `create`. -* **url** [String, optional]: URL of a release to download or local directory path when version is `create`. Works with CLI v2. Example: `https://bosh.io/d/github.com/cloudfoundry/syslog-release?v=11`. -* **sha1** [String, optional]: SHA1 of asset referenced via URL. Works with CLI v2. Example: `332ac15609b220a3fdf5efad0e0aa069d8235788`. -* **stemcell** [Hash, optional]: When `url` refers to a compiled release, the stemcell on which it was compiled (recommended when `url` refers to a compiled release). - * **os** [String, required]: Operating system of the stemcell. Example: `ubuntu-xenial`. - * **version** [String, required]: Version of the stemcell. Example: `97.18`. -* **exported_from** [Array, optional]: Require the release be deployed from previously-compiled releases of specific stemcell versions. - * **os** [String, required]: Operating system of the stemcell. Example: `ubuntu-xenial`. - * **version** [String, required]: Version of the stemcell. Example: `97.18`. +- **name** [String, required]: Name of a release used in the deployment. +- **version** [String, required]: The version of the release to use. Version can be `latest` or `create`. +- **url** [String, optional]: URL of a release to download or local directory path when version is `create`. Works with CLI v2. Example: `https://bosh.io/d/github.com/cloudfoundry/syslog-release?v=11`. +- **sha1** [String, optional]: SHA1 of asset referenced via URL. Works with CLI v2. Example: `332ac15609b220a3fdf5efad0e0aa069d8235788`. +- **stemcell** [Hash, optional]: When `url` refers to a compiled release, the stemcell on which it was compiled (recommended when `url` refers to a compiled release). + - **os** [String, required]: Operating system of the stemcell. Example: `ubuntu-xenial`. + - **version** [String, required]: Version of the stemcell. Example: `97.18`. +- **exported_from** [Array, optional]: Require the release be deployed from previously-compiled releases of specific stemcell versions. + - **os** [String, required]: Operating system of the stemcell. Example: `ubuntu-xenial`. + - **version** [String, required]: Version of the stemcell. Example: `97.18`. See [Release URLs](release-urls.md) for more details. @@ -119,14 +124,15 @@ releases: ``` --- + ## Stemcells Block {: #stemcells } **stemcells** [Array, required]: The name and version of each stemcell in the deployment. -* **alias** [String, required]: Name of a stemcell used in the deployment -* **os** [String, optional]: Operating system of a matching stemcell. Example: `ubuntu-xenial`. -* **version** [String, required]: The version of a matching stemcell. Version can be `latest`. -* **name** [String, optional]: Full name of a matching stemcell. Either `name` or `os` keys can be specified. +- **alias** [String, required]: Name of a stemcell used in the deployment +- **os** [String, optional]: Operating system of a matching stemcell. Example: `ubuntu-xenial`. +- **version** [String, required]: The version of a matching stemcell. Version can be `latest`. +- **name** [String, optional]: Full name of a matching stemcell. Either `name` or `os` keys can be specified. !!! tip "IaaS-agnostic Configuration" To ensure your manifest can easily be used across multiple IaaSes, prefer using `os` instead of the IaaS-specific `name` field. Note: unlike `releases`, this does not support automatic downloading of a stemcell through a `url` field. @@ -144,30 +150,31 @@ stemcells: ``` --- + ## Update Block {: #update } **update** [Hash, required]: This specifies instance update properties. These properties control how BOSH updates instances during the deployment. -* **canaries** [Integer or Percentage, required]: The number of [canary](terminology.md#canary) instances. -* **max\_in\_flight** [Integer or Percentage, required]: The maximum number of non-canary instances to update in parallel within an availability zone. Updates will not begin in another availability zone until all VMs are updated in the current availability zone. - * If the `max_in_flight` is a percentage, the minimum `max_in_flight` will never fall below 1. -* **canary\_watch\_time** [Integer or Range, required]: Only applies to monit start operation. - * If the `canary_watch_time` is an integer, the Director sleeps for that many milliseconds, then checks whether the canary instances are healthy. - * If the `canary_watch_time` is a range (low-high), the Director: - * Waits for `low` milliseconds - * Waits until instances are healthy or `high` milliseconds have passed since instances started updating -* **update\_watch\_time** [Integer or Range, required]: Only applies to monit start operation. - * If the `update_watch_time` is an integer, the Director sleeps for that many milliseconds, then checks whether the instances are healthy. - * If the `update_watch_time` is a range (low-high), the Director: - * Waits for `low` milliseconds - * Waits until instances are healthy or `high` milliseconds have passed since instances started updating -* **serial** [Boolean, optional]: If disabled (set to `false`), instance groups will be deployed in parallel, otherwise - sequentially. Instances within a group will still follow `canary` and `max_in_flight` configuration. Defaults to `true`. -* **vm_strategy** [String, optional]: Influence how instances are updated when their VM needs to be recreated due to IaaS-related changes. Choose from one of the following strategies. Defaults to `delete-create`. Introduced in [bosh/267.2](https://github.com/cloudfoundry/bosh/releases/tag/v267.2). - * `delete-create` - fully stop processes and delete the VM before creating and provisioning a new VM. - * `create-swap-delete` - create and provision a new VM before stopping processes and transferring responsibilities from the old VM. Some configurations (e.g. static IPs) are not supported and will automatically revert to the `delete-create` strategy. -* **initial_deploy_az_update_strategy** [String, optional]: Configure how instances are updated across multiple availability zones on the first deploy of a new deployment. Instances will still be updated with respect to `canaries` and `max_in_flight`. Defaults to `serial`. **Note**: VMs are all created in parallel regardless. - * `serial` - Instances in each instance group will be updated one availability zone at a time. - * `parallel` - Instances in each instance group will be updated all at once, regardless of availability zone. +- **canaries** [Integer or Percentage, required]: The number of [canary](terminology.md#canary) instances. +- **max\_in\_flight** [Integer or Percentage, required]: The maximum number of non-canary instances to update in parallel within an availability zone. Updates will not begin in another availability zone until all VMs are updated in the current availability zone. + - If the `max_in_flight` is a percentage, the minimum `max_in_flight` will never fall below 1. +- **canary\_watch\_time** [Integer or Range, required]: Only applies to monit start operation. + - If the `canary_watch_time` is an integer, the Director sleeps for that many milliseconds, then checks whether the canary instances are healthy. + - If the `canary_watch_time` is a range (low-high), the Director: + - Waits for `low` milliseconds + - Waits until instances are healthy or `high` milliseconds have passed since instances started updating +- **update\_watch\_time** [Integer or Range, required]: Only applies to monit start operation. + - If the `update_watch_time` is an integer, the Director sleeps for that many milliseconds, then checks whether the instances are healthy. + - If the `update_watch_time` is a range (low-high), the Director: + - Waits for `low` milliseconds + - Waits until instances are healthy or `high` milliseconds have passed since instances started updating +- **serial** [Boolean, optional]: If disabled (set to `false`), instance groups will be deployed in parallel, otherwise - sequentially. Instances within a group will still follow `canary` and `max_in_flight` configuration. Defaults to `true`. +- **vm_strategy** [String, optional]: Influence how instances are updated when their VM needs to be recreated due to IaaS-related changes. Choose from one of the following strategies. Defaults to `delete-create`. Introduced in [bosh/267.2](https://github.com/cloudfoundry/bosh/releases/tag/v267.2). + - `delete-create` - fully stop processes and delete the VM before creating and provisioning a new VM. + - `create-swap-delete` - create and provision a new VM before stopping processes and transferring responsibilities from the old VM. Some configurations (e.g. static IPs) are not supported and will automatically revert to the `delete-create` strategy. +- **initial_deploy_az_update_strategy** [String, optional]: Configure how instances are updated across multiple availability zones on the first deploy of a new deployment. Instances will still be updated with respect to `canaries` and `max_in_flight`. Defaults to `serial`. **Note**: VMs are all created in parallel regardless. + - `serial` - Instances in each instance group will be updated one availability zone at a time. + - `parallel` - Instances in each instance group will be updated all at once, regardless of availability zone. See [job lifecycle](job-lifecycle.md) for more details on startup/shutdown procedure within each VM. @@ -192,59 +199,60 @@ update: ``` --- + ## Instance Groups Block {: #instance-groups } **instance_groups** [Array, required]: Specifies the mapping between release [jobs](terminology.md#job) and instance groups. -* **name** [String, required]: A unique name used to identify and reference instance group. -* **azs** [Array, required]: List of AZs associated with this instance group (should only be used when using [first class AZs](azs.md)). Example: `[z1, z2]`. -* **instances** [Integer, required]: The number of instances in this group. Each instance is a VM. -* **jobs** [Array, required]: Specifies the name and release of jobs that will be installed on each instance. - * **name** [String, required]: The job name - * **release** [String, required]: The release where the job exists - * **consumes** [Hash, optional]: Links consumed by the job. [Read more about link configuration](links.md#deployment) - * **provides** [Hash, optional]: Links provided by the job. [Read more about link configuration](links.md#deployment) - * **properties** [Hash, optional]: Specifies job properties. Properties allow BOSH to configure jobs to a specific environment. `properties` defined in a Job block are accessible only to that job. If `properties` are specified, only these will be provided to the job and instance group properties will be ignored. -* **vm_type** [String, required]: A valid VM type name from the cloud config. Alternatively you can specify `vm_resources` key. -* **vm_extensions** [Array, optional]: A valid list of VM extension names from the cloud config. -* **vm_resources** [Hash, optional]: Specifies generic VM resources such as CPU, RAM and disk size that are automatically translated into correct VM cloud properties to determine VM size. VM size is determined on best effort basis as some IaaSes may not support exact size configuration. Available in bosh-release v264+. - * **cpu** [Integer, required]: Number of CPUs. - * **ram** [Integer, required]: Amount of RAM in MB. - * **ephemeral\_disk\_size** [Integer, required]: Ephemeral disk size in MB. -* **stemcell** [String, required]: A valid stemcell alias from the Stemcells Block. -* **persistent\_disk** [Integer, optional]: Persistent disk size in MB. Alternatively you can specify `persistent_disk_type` key. [Read more about persistent disks](persistent-disks.md) -* **persistent\_disk\_type** [String, optional]: A valid disk type name from the cloud config. [Read more about persistent disks](persistent-disks.md) -* **networks** [Array, required]: Specifies the networks this instance requires. Each network can have the following properties specified: - * **name** [String, required]: A valid network name from the cloud config. - * **static_ips** [Array, optional]: Array of IP addresses reserved for the instances on the network. - * **default** [Array, optional]: Specifies which network components (DNS, Gateway) BOSH populates by default from this network. This property is required if more than one network is specified. - * **nic_group** [Integer, optional]: Defines a nic_group. Networks having the same nic_group will be assigned to the same network interface card (if possible). Find supported CPIs [here](networks.md#cpi-limitations). -* **update** [Hash, optional]: Specific update settings for this instance group. Use this to override [global job update settings](#update) on a per-instance-group basis. -* **migrated_from** [Array, optional]: Specific migration settings for this instance group. Use this to [rename and/or migrate instance groups](migrated-from.md). -* **lifecycle** [String, optional]: Specifies the kind of workload the instance group represents. Valid values are `service` and `errand`; defaults to `service`. A `service` runs indefinitely and restarts if it fails. An `errand` starts with a manual trigger and does not restart if it fails. -* **properties** [Hash, optional]: Specifies instance group properties. Deprecated in favor of job level properties and links. -* **tags** [Hash, optional]: Specifies instance group tags. Specifies key value pairs to be sent to the CPI for VM tagging. Combined with deployment tags during VM creation. Available in bosh-release v277.4.0+. -* **env** [Hash, optional]: Specifies advanced BOSH Agent configuration for each instance in the group. [Read more about the agent.](agent-cpi-interactions.md#agent-settings-format) - * **persistent_disk_fs** [String, optional]: Filesystem type to use when formatting persistent disk. Supported values: `ext4`, `xfs`. Default is currently set to `ext4` but may change. [See details](persistent-disk-fs.md) - * **persistent_disk_mount_options** [Array of strings, optional]: Mount options when mounting persistent disk. Example: `["noatime"]`. - * **bosh** [Hash, optional]: - * **password** [String, optional]: Crypted password for `vcap/root` user (will be placed into /etc/shadow on Linux). - * **authorized_keys** [Array, optional]: Public keys for the `vcap` user, will be placed in `.ssh/authorized_keys`. This value can be used to enable SSH access onto VMs in this instance group. +- **name** [String, required]: A unique name used to identify and reference instance group. +- **azs** [Array, required]: List of AZs associated with this instance group (should only be used when using [first class AZs](azs.md)). Example: `[z1, z2]`. +- **instances** [Integer, required]: The number of instances in this group. Each instance is a VM. +- **jobs** [Array, required]: Specifies the name and release of jobs that will be installed on each instance. + - **name** [String, required]: The job name + - **release** [String, required]: The release where the job exists + - **consumes** [Hash, optional]: Links consumed by the job. [Read more about link configuration](links.md#deployment) + - **provides** [Hash, optional]: Links provided by the job. [Read more about link configuration](links.md#deployment) + - **properties** [Hash, optional]: Specifies job properties. Properties allow BOSH to configure jobs to a specific environment. `properties` defined in a Job block are accessible only to that job. If `properties` are specified, only these will be provided to the job and instance group properties will be ignored. +- **vm_type** [String, required]: A valid VM type name from the cloud config. Alternatively you can specify `vm_resources` key. +- **vm_extensions** [Array, optional]: A valid list of VM extension names from the cloud config. +- **vm_resources** [Hash, optional]: Specifies generic VM resources such as CPU, RAM and disk size that are automatically translated into correct VM cloud properties to determine VM size. VM size is determined on best effort basis as some IaaSes may not support exact size configuration. Available in bosh-release v264+. + - **cpu** [Integer, required]: Number of CPUs. + - **ram** [Integer, required]: Amount of RAM in MB. + - **ephemeral\_disk\_size** [Integer, required]: Ephemeral disk size in MB. +- **stemcell** [String, required]: A valid stemcell alias from the Stemcells Block. +- **persistent\_disk** [Integer, optional]: Persistent disk size in MB. Alternatively you can specify `persistent_disk_type` key. [Read more about persistent disks](persistent-disks.md) +- **persistent\_disk\_type** [String, optional]: A valid disk type name from the cloud config. [Read more about persistent disks](persistent-disks.md) +- **networks** [Array, required]: Specifies the networks this instance requires. Each network can have the following properties specified: + - **name** [String, required]: A valid network name from the cloud config. + - **static_ips** [Array, optional]: Array of IP addresses reserved for the instances on the network. + - **default** [Array, optional]: Specifies which network components (DNS, Gateway) BOSH populates by default from this network. This property is required if more than one network is specified. + - **nic_group** [Integer, optional]: Defines a nic_group. Networks having the same nic_group will be assigned to the same network interface card (if possible). Find supported CPIs [here](networks.md#cpi-limitations). +- **update** [Hash, optional]: Specific update settings for this instance group. Use this to override [global job update settings](#update) on a per-instance-group basis. +- **migrated_from** [Array, optional]: Specific migration settings for this instance group. Use this to [rename and/or migrate instance groups](migrated-from.md). +- **lifecycle** [String, optional]: Specifies the kind of workload the instance group represents. Valid values are `service` and `errand`; defaults to `service`. A `service` runs indefinitely and restarts if it fails. An `errand` starts with a manual trigger and does not restart if it fails. +- **properties** [Hash, optional]: Specifies instance group properties. Deprecated in favor of job level properties and links. +- **tags** [Hash, optional]: Specifies instance group tags. Specifies key value pairs to be sent to the CPI for VM tagging. Combined with deployment tags during VM creation. Available in bosh-release v277.4.0+. +- **env** [Hash, optional]: Specifies advanced BOSH Agent configuration for each instance in the group. [Read more about the agent.](agent-cpi-interactions.md#agent-settings-format) + - **persistent_disk_fs** [String, optional]: Filesystem type to use when formatting persistent disk. Supported values: `ext4`, `xfs`. Default is currently set to `ext4` but may change. [See details](persistent-disk-fs.md) + - **persistent_disk_mount_options** [Array of strings, optional]: Mount options when mounting persistent disk. Example: `["noatime"]`. + - **bosh** [Hash, optional]: + - **password** [String, optional]: Crypted password for `vcap/root` user (will be placed into /etc/shadow on Linux). + - **authorized_keys** [Array, optional]: Public keys for the `vcap` user, will be placed in `.ssh/authorized_keys`. This value can be used to enable SSH access onto VMs in this instance group. * **keep\_root\_password** [Boolean, optional]: Keep password for `root` and only change password for `vcap`. Default: `false`. - * **remove\_dev\_tools** [Boolean, optional]: Remove [compilers and dev tools](https://github.com/cloudfoundry/bosh-linux-stemcell-builder/blob/master/stemcell_builder/stages/dev_tools_config/assets/generate_dev_tools_file_list.sh) on non-compilation VMs. Default: `false`. - * **remove\_static\_libraries** [Boolean, optional]: Remove [static libraries](https://github.com/cloudfoundry/bosh-linux-stemcell-builder/blob/master/stemcell_builder/stages/static_libraries_config/assets/static_libraries_list.txt) on non-compilation VMs. Default: `false`. - * **swap\_size** [Integer, optional]: Size of swap partition in MB to create. Set this to 0 to avoid having a swap partition created. Default: RAM size of used VM type up to half of the ephemeral disk size. - * **ipv6** [Hash, optional]: - * **enable** [Boolean, optional]: Force IPv6 enabled in kernel (this configuration is not necessary if one of the VM addresses is IPv6). Default: `false`. - * **job_dir** [Hash, optional]: - * **tmpfs** [Boolean, optional]: Mount all directories which contain [rendered job templates on tmpfs](creds-tmpfs.md). Default: `false`. - * **tmpfs_size** [String, optional]: The size of the tmpfs mount. Accepted values are a number of bytes or a number with a suffix e.g. `64m` or `1g`. Default: `100m`. - * **run_dir** [Hash, optional]: - * **tmpfs_size** [String, optional]: The size of the tmpfs mount `/var/vcap/data/sys/run`. Accepted values are a number of bytes or a number with a suffix e.g. `64m` or `1g` .Default: `16m`. - * **agent** [Hash, optional]: - * **settings** [Hash, optional]: - * **tmpfs** [Boolean, optional]: Mount agent settings on a [tmpfs](creds-tmpfs.md) directory. Default: `false`. - * **ntp** [Array, optional]: list of [ntp servers to synchronize to](ntp-config.md). Default: director-configured ntp servers. + - **remove\_dev\_tools** [Boolean, optional]: Remove [compilers and dev tools](https://github.com/cloudfoundry/bosh-linux-stemcell-builder/blob/master/stemcell_builder/stages/dev_tools_config/assets/generate_dev_tools_file_list.sh) on non-compilation VMs. Default: `false`. + - **remove\_static\_libraries** [Boolean, optional]: Remove [static libraries](https://github.com/cloudfoundry/bosh-linux-stemcell-builder/blob/master/stemcell_builder/stages/static_libraries_config/assets/static_libraries_list.txt) on non-compilation VMs. Default: `false`. + - **swap\_size** [Integer, optional]: Size of swap partition in MB to create. Set this to 0 to avoid having a swap partition created. Default: RAM size of used VM type up to half of the ephemeral disk size. + - **ipv6** [Hash, optional]: + - **enable** [Boolean, optional]: Force IPv6 enabled in kernel (this configuration is not necessary if one of the VM addresses is IPv6). Default: `false`. + - **job_dir** [Hash, optional]: + - **tmpfs** [Boolean, optional]: Mount all directories which contain [rendered job templates on tmpfs](creds-tmpfs.md). Default: `false`. + - **tmpfs_size** [String, optional]: The size of the tmpfs mount. Accepted values are a number of bytes or a number with a suffix e.g. `64m` or `1g`. Default: `100m`. + - **run_dir** [Hash, optional]: + - **tmpfs_size** [String, optional]: The size of the tmpfs mount `/var/vcap/data/sys/run`. Accepted values are a number of bytes or a number with a suffix e.g. `64m` or `1g` .Default: `16m`. + - **agent** [Hash, optional]: + - **settings** [Hash, optional]: + - **tmpfs** [Boolean, optional]: Mount agent settings on a [tmpfs](creds-tmpfs.md) directory. Default: `false`. + - **ntp** [Array, optional]: list of [ntp servers to synchronize to](ntp-config.md). Default: director-configured ntp servers. Example: @@ -282,6 +290,7 @@ instance_groups: ``` --- + ## Addons Block {: #addons } !!! note @@ -295,7 +304,7 @@ Unlike addons specified in a runtime config, addons specified in the deployment Example: -``` +```yaml addons: - name: logging jobs: @@ -306,27 +315,29 @@ addons: ``` --- + ## Properties Block {: #properties } **properties** [Hash, optional]: Describes global properties. Deprecated in favor of job level properties and links. --- + ## Variables Block {: #variables } **variables** [Array, optional]: Describes variables. -* **name** [String, required]: Unique name used to identify a variable. Example: `admin_password` -* **type** [String, required]: Type of a variable. Currently supported variable types are `certificate`, `password`, `rsa`, and `ssh`. -* **update_mode** [String, optional]: Update mode to use when generating credentials. Currently supported update modes are `no-overwrite`, `overwrite`, and `converge`. Defaults to `no-overwrite`. For further information about the behavior of the update modes, please refer to the [CredHub documentation](https://docs.cloudfoundry.org/api/credhub/version/2.9/#_overwriting_credential_values). -* **update** [Hash, optional]: Available in bosh-release 279+. - * **strategy** [String, optional]: Controls when BOSH retrieves the latest version for the variable from the config server during a deploy. Currently supported strategies are `on-deploy`, and `on-stemcell-change`. Defaults to `on-deploy`. - * `on-deploy`: BOSH will look up the latest variable value on each deploy. - * `on-stemcell-change`: BOSH will only look up the latest variable value when a deploy is occurring that updates all stemcells for the deployment. If one or more stemcells are not being updated, then BOSH will continue to use the previously deployed value for the variable. -* **options** [Hash, optional]: Specifies generation options used for generating variable value if variable is not found. Example: `{is_ca: true, common_name: some-ca}` +- **name** [String, required]: Unique name used to identify a variable. Example: `admin_password` +- **type** [String, required]: Type of a variable. Currently supported variable types are `certificate`, `password`, `rsa`, and `ssh`. +- **update_mode** [String, optional]: Update mode to use when generating credentials. Currently supported update modes are `no-overwrite`, `overwrite`, and `converge`. Defaults to `no-overwrite`. For further information about the behavior of the update modes, please refer to the [CredHub documentation](https://docs.cloudfoundry.org/api/credhub/version/2.9/#_overwriting_credential_values). +- **update** [Hash, optional]: Available in bosh-release 279+. + - **strategy** [String, optional]: Controls when BOSH retrieves the latest version for the variable from the config server during a deploy. Currently supported strategies are `on-deploy`, and `on-stemcell-change`. Defaults to `on-deploy`. + - `on-deploy`: BOSH will look up the latest variable value on each deploy. + - `on-stemcell-change`: BOSH will only look up the latest variable value when a deploy is occurring that updates all stemcells for the deployment. If one or more stemcells are not being updated, then BOSH will continue to use the previously deployed value for the variable. +- **options** [Hash, optional]: Specifies generation options used for generating variable value if variable is not found. Example: `{is_ca: true, common_name: some-ca}` Example: -``` +```yaml variables: - name: admin_password type: password @@ -349,6 +360,7 @@ See [Variable Types](variable-types.md) for more details about CLI-supported generation for variables. --- + ## Tags Block {: #tags } **tags** [Hash, optional]: Specifies key value pairs to be sent to the CPI for VM and disk tagging. Combined with runtime config level tags during the deploy. Available in bosh-release v258+. diff --git a/content/mbus-ssl-rotation.md b/content/mbus-ssl-rotation.md index 641dd4169..5c477e3d5 100644 --- a/content/mbus-ssl-rotation.md +++ b/content/mbus-ssl-rotation.md @@ -3,12 +3,12 @@ Bootstrap SSL certificate needs to be removed from the `creds.yml` and re-created later with `create-env`. Since the signing CA is the Director's default CA and remains unchanged, no updates are required on the CLI side. -### Preconditions +## Preconditions -* Director is in a healthy state and there are no new deployments in progress. -* These instructions must be adapted if used with ops files overwriting the variables used in this procedure (i.e. bosh-lite). +- Director is in a healthy state and there are no new deployments in progress. +- These instructions must be adapted if used with ops files overwriting the variables used in this procedure (i.e. bosh-lite). -### Step 1: Remove mbus SSL from `creds.yml` {: #step-1} +## Step 1: Remove mbus SSL from `creds.yml` {: #step-1} ```shell bosh interpolate ./creds.yml \ @@ -25,9 +25,9 @@ Ops file `remove-mbus-ssl.yml` path: /mbus_bootstrap_ssl? ``` -* This will remove the `mbus_bootstrap_ssl` from the `creds.yml`, causing the next `create-env` to create a new one. +- This will remove the `mbus_bootstrap_ssl` from the `creds.yml`, causing the next `create-env` to create a new one. -### Step 2: Redeploy the Director with a new mbus SSL certificate {: #step-2} +## Step 2: Redeploy the Director with a new mbus SSL certificate {: #step-2} ```shell bosh create-env ~/workspace/bosh-deployment/bosh.yml \ @@ -38,4 +38,4 @@ bosh create-env ~/workspace/bosh-deployment/bosh.yml \ -v ... additional vars ``` -* This adds a new mbus SSL certificate to `creds.yml`. +- This adds a new mbus SSL certificate to `creds.yml`. diff --git a/content/migrated-from.md b/content/migrated-from.md index 6923b14f1..5ae65edbb 100644 --- a/content/migrated-from.md +++ b/content/migrated-from.md @@ -1,3 +1,5 @@ +# Migrating Instance Groups + Occasionally, it's convenient to rename one or more instance groups as their purpose changes or as better names are found. In most cases it's desirable to maintain existing persistent data by keeping existing persistent disks. Previously, the CLI provided the `rename job` command to rename a specific instance group one at a time. That approach worked OK in non-automated, non-frequently updated environments, but it was inconvenient for automated, frequently updated environments. As a replacement, the `migrated_from` directive was added to allow renames to happen in a more systematic way. @@ -5,14 +7,16 @@ Previously, the CLI provided the `rename job` command to rename a specific insta Additionally `migrated_from` directive can be used to migrate instance groups to use first class AZs. --- + ## Schema {: #schema } **migrated_from** [Array, required]: The name and AZ of each instance group that should be used to form new instance group. -* **name** [String, required]: Name of an instance group that used to exist in the manifest. -* **az** [String, optional]: Availability zone that was used for the named instance group. This key is optional for instance groups that used first class AZs (via `azs` key). If first class AZ was not used, then this key must specify first class AZ that matches actual IaaS AZ configuration. +- **name** [String, required]: Name of an instance group that used to exist in the manifest. +- **az** [String, optional]: Availability zone that was used for the named instance group. This key is optional for instance groups that used first class AZs (via `azs` key). If first class AZ was not used, then this key must specify first class AZ that matches actual IaaS AZ configuration. --- + ## Renaming Instance Groups {: #rename } 1. Given follow deployment instance group `etcd`: @@ -50,6 +54,7 @@ Additionally `migrated_from` directive can be used to migrate instance groups to 1. Deploy. --- + ## Migrating Instance Groups (to first class AZs) {: #migrate } Before the introduction of first class AZs, each instance group was associated with a resource pool that typically defined some CPI specific AZ configuration in its `cloud_properties`. Typically there would be multiple instance groups that mostly differed by their name, for example `etcd_z1` and `etcd_z2`. With first class AZs, multiple instance groups typically should be collapsed to simplify the deployment. diff --git a/content/monitoring.md b/content/monitoring.md index a0d735e82..0f93a2ad2 100644 --- a/content/monitoring.md +++ b/content/monitoring.md @@ -1,6 +1,9 @@ +# Monitoring + BOSH monitors deployed VMs and release jobs' processes on those VMs via the Health Monitor and the help of the Agent, and Monit. --- + ## VMs {: #vm } [The Health Monitor](bosh-components.md#health-monitor) continuously checks presence of the deployed VMs. The Agent on each VM produces a heartbeat every minute and sends it to the Health Monitor over [NATS](bosh-components.md#nats). @@ -27,27 +30,32 @@ Resurrector plugin continuously cross-references VMs expected to be running agai See [Automatic repair with Resurrector](resurrector.md) for details. --- + ## Processes on VMs {: #process } Release jobs' process monitoring on each VM is done with the help of [Monit, version 5.2.5](https://web.archive.org/web/20110816041503/https://mmonit.com/monit/documentation/monit.html). Monit continuously monitors presence of the configured release jobs' processes and restarts processes that are not found. Process restarts, failures, etc. are reported to the Agent which in turn reports them as alerts to the Health Monitor. Each Health Monitor plugin is given an opportunity to act on each alert. --- + ## SSH Events {: #ssh } The Agent on each VM sends an alert when someone/something tries to log into the system via SSH. Successful and failed attempts are recorded. --- + ## Deploy Events {: #deploy } The Director sends an alert when a deployment starts, successfully completes or errors. --- + ## NATS {: #nats } -NATS monitoring requires BOSH version `268.4+` and can be enabled by setting the director property `enable_metrics_endpoint` to `true` as described [here](https://bosh.io/jobs/nats?source=github.com/cloudfoundry/bosh#p%3dnats.enable_metrics_endpoint). +NATS monitoring requires BOSH version `268.4+` and can be enabled by setting the director property `enable_metrics_endpoint` to `true` as described in the [NATS job documentation](https://bosh.io/jobs/nats?source=github.com/cloudfoundry/bosh#p%3dnats.enable_metrics_endpoint). The monitoring endpoint is exposed locally on the director VM. More information about it can be found in the [NATS documentation](https://docs.nats.io/running-a-nats-service/nats_admin/monitoring). --- + ## NGINX {: #nginx } Nginx monitoring requires BOSH version `268.3+` and can be enabled by setting `enable_metrics_endpoint` to `true`. diff --git a/content/nats-ca-rotation.md b/content/nats-ca-rotation.md index db98e5f22..570de29c9 100644 --- a/content/nats-ca-rotation.md +++ b/content/nats-ca-rotation.md @@ -6,10 +6,10 @@ The procedure below rotates the NATS CA and NATS related certificates across the ### Preconditions {: #preconditions } -* Director is in a healthy state. -* All VMs are in `running` state in all deployments. See [below](#expired) if your VMs are unresponsive. -* Take note of any **ignored** VMs. They will be omitted from the VM redeploy steps. -* Former Director versions (prior to 271.12) and stemcells (prior to Bionic +- Director is in a healthy state. +- All VMs are in `running` state in all deployments. See [below](#expired) if your VMs are unresponsive. +- Take note of any **ignored** VMs. They will be omitted from the VM redeploy steps. +- Former Director versions (prior to 271.12) and stemcells (prior to Bionic 1.36 or Windows 2019.41) need to recreate VMs as part of the redeploy steps ([step 2](#step-2) and [step 4](#step-4)). @@ -42,11 +42,11 @@ bosh create-env ~/workspace/bosh-deployment/bosh.yml \ -v ... additional vars ``` -* Adds new variables to generate the new NATS CA and the corresponding NATS related certificates signed by it. Please note that not all of these newly generated certificates will be used in this step, as some of them will be used in the following steps. -* The director and health monitor jobs are given two CA certificates to trust when communicating with the NATS server. This is done through the concatenation of the old and new NATS CAs: `((nats_server_tls.ca))((nats_server_tls_2.ca))`. This allows the director and health monitor to trust certificates presented by the NATS server that can be either signed by the new or old CAs. -* The director and health monitor jobs are updated to use new client certificates that were generated by the new CA. These client certs are used for the Mutual TLS communication with the NATS server. -* The NATS server continues to use the old certificates (signed by old NATS CA) to serve TLS connections. NATS server is given the concatenated CAs from above to verify client certificates (for mTLS) signed by both old CA and new CA. -* Each VM/agent continues to use the old client certificates to communicate with the NATS server. +- Adds new variables to generate the new NATS CA and the corresponding NATS related certificates signed by it. Please note that not all of these newly generated certificates will be used in this step, as some of them will be used in the following steps. +- The director and health monitor jobs are given two CA certificates to trust when communicating with the NATS server. This is done through the concatenation of the old and new NATS CAs: `((nats_server_tls.ca))((nats_server_tls_2.ca))`. This allows the director and health monitor to trust certificates presented by the NATS server that can be either signed by the new or old CAs. +- The director and health monitor jobs are updated to use new client certificates that were generated by the new CA. These client certs are used for the Mutual TLS communication with the NATS server. +- The NATS server continues to use the old certificates (signed by old NATS CA) to serve TLS connections. NATS server is given the concatenated CAs from above to verify client certificates (for mTLS) signed by both old CA and new CA. +- Each VM/agent continues to use the old client certificates to communicate with the NATS server. !!! warning In the below operations file `add-new-ca.yml`, the `nats_server_tls_2` certificate is generated with the `internal_ip` as the only “Subject Alternative Name”. Please remember to add any other SANs that maybe necessary to your environment. @@ -147,10 +147,10 @@ bosh create-env ~/workspace/bosh-deployment/bosh.yml \ -v ... additional vars ``` -* `nats.tls.ca` property is updated to remove the old CA from the concatenated CAs. -* The director and health monitor continue to only use new client certificates (for mTLS) that were signed by the new NATS CA. Also, in this step the director and health monitor will start to **ONLY** trust NATS server certificates that were signed by the new CA. -* The NATS server is updated to use a new certificate (used to serve TLS connections) signed by the new NATS CA. Also, in this step the NATS server will start to **ONLY** trust client certificates (for mTLS) that were signed by the new CA. -* All components now communicate using the new CA. +- `nats.tls.ca` property is updated to remove the old CA from the concatenated CAs. +- The director and health monitor continue to only use new client certificates (for mTLS) that were signed by the new NATS CA. Also, in this step the director and health monitor will start to **ONLY** trust NATS server certificates that were signed by the new CA. +- The NATS server is updated to use a new certificate (used to serve TLS connections) signed by the new NATS CA. Also, in this step the NATS server will start to **ONLY** trust client certificates (for mTLS) that were signed by the new CA. +- All components now communicate using the new CA. `remove-old-ca.yml` @@ -185,7 +185,6 @@ bosh create-env ~/workspace/bosh-deployment/bosh.yml \ private_key: ((nats_clients_health_monitor_tls_2.private_key)) ``` - ### Step 4: Redeploy all VMs, for each deployment {: #step-4} Redeploying all VMs will remove the old NATS CA reference from their agent settings. diff --git a/content/networks.md b/content/networks.md index 00de6bcfd..51e9599c4 100644 --- a/content/networks.md +++ b/content/networks.md @@ -1,15 +1,17 @@ +# Networks + A BOSH network is an IaaS-agnostic representation of the networking layer. The Director is responsible for configuring each instance group's networks with the help of the BOSH Agent and the IaaS. Networking configuration is usually assigned at the boot of the VM and/or when network configuration changes in the deployment manifest for already-running instance groups. There are three types of networks that BOSH supports: -* **manual**: The Director decides how to assign IPs to each instance based on the specified network subnets in the deployment manifest -* **dynamic**: The Director defers IP selection to the IaaS -* **vip**: The Director allows one-off IP assignments to specific instances to enable flexible IP routing (e.g. elastic IP) +- **manual**: The Director decides how to assign IPs to each instance based on the specified network subnets in the deployment manifest +- **dynamic**: The Director defers IP selection to the IaaS +- **vip**: The Director allows one-off IP assignments to specific instances to enable flexible IP routing (e.g. elastic IP) Each type of network supports one or both IP reservation types: -* **static**: IP is explicitly requested by the user in the deployment manifest -* **automatic**: IP is selected automatically based on the network type +- **static**: IP is explicitly requested by the user in the deployment manifest +- **automatic**: IP is selected automatically based on the network type | | Manual network | Dynamic network | VIP network | | ----------------------- | ------------------ | --------------- | ----------- | @@ -79,18 +81,18 @@ Each manual network attached to an instance is typically represented as its own Schema for manual network definition: -* **name** [String, required]: Name used to reference this network configuration -* **type** [String, required]: Value should be `manual` -* **subnets** [Array, required]: Lists subnets in this network - * **range** [String, required]: Subnet IP range that includes all IPs from this subnet - * **gateway** [String, required]: Subnet gateway IP - * **dns** [Array, optional]: DNS IP addresses for this subnet - * **reserved** [Array, optional]: Array of reserved IPs and/or IP ranges. BOSH does not assign IPs from this range to any VM - * **static** [Array, optional]: Array of static IPs and/or IP ranges. BOSH assigns IPs from this range to instances requesting static IPs. Only IPs specified here can be used for static IP reservations. - * **prefix** [String, optional]: Size of the prefix BOSH will assign to VMs. Networks that have this property set cannot be used by BOSH itself; therefore, if this is set, a secondary network needs to be attached. Supported from director version `v282.1.0` and stemcell `Ubuntu Jammy v1.943`. Find more information in the [Prefix Delegation](#prefix-delegation) section. - * **az** [String, optional]: AZ associated with this subnet (should only be used when using [first class AZs](azs.md)). Example: `z1`. Available in v241+. - * **azs** [Array, optional]: List of AZs associated with this subnet (should only be used when using [first class AZs](azs.md)). Example: `[z1, z2]`. Available in v241+. - * **cloud_properties** [Hash, optional]: Describes any IaaS-specific properties for the subnet. Default is `{}` (empty Hash). +- **name** [String, required]: Name used to reference this network configuration +- **type** [String, required]: Value should be `manual` +- **subnets** [Array, required]: Lists subnets in this network + - **range** [String, required]: Subnet IP range that includes all IPs from this subnet + - **gateway** [String, required]: Subnet gateway IP + - **dns** [Array, optional]: DNS IP addresses for this subnet + - **reserved** [Array, optional]: Array of reserved IPs and/or IP ranges. BOSH does not assign IPs from this range to any VM + - **static** [Array, optional]: Array of static IPs and/or IP ranges. BOSH assigns IPs from this range to instances requesting static IPs. Only IPs specified here can be used for static IP reservations. + - **prefix** [String, optional]: Size of the prefix BOSH will assign to VMs. Networks that have this property set cannot be used by BOSH itself; therefore, if this is set, a secondary network needs to be attached. Supported from director version `v282.1.0` and stemcell `Ubuntu Jammy v1.943`. Find more information in the [Prefix Delegation](#prefix-delegation) section. + - **az** [String, optional]: AZ associated with this subnet (should only be used when using [first class AZs](azs.md)). Example: `z1`. Available in v241+. + - **azs** [Array, optional]: List of AZs associated with this subnet (should only be used when using [first class AZs](azs.md)). Example: `[z1, z2]`. Available in v241+. + - **cloud_properties** [Hash, optional]: Describes any IaaS-specific properties for the subnet. Default is `{}` (empty Hash). Example cloud config: @@ -159,16 +161,16 @@ networks: In this example, the Director divides the `/24` subnet into `/28` subnets to assign to VMs (as soon as the network is referenced by a deployment manifest). The next available base address of the prefix within the subnet range is calculated for each assignment. For example, the first three base addresses would be: -* `10.10.0.0/28` <-- bosh will not assign this one, because the range contains reserved ip addresses (gateway etc.) -* `10.10.0.16/28` <-- first prefix that will be assigned to a vm -* `10.10.0.32/28` +- `10.10.0.0/28` <-- bosh will not assign this one, because the range contains reserved ip addresses (gateway etc.) +- `10.10.0.16/28` <-- first prefix that will be assigned to a vm +- `10.10.0.32/28` The IP and prefix information will get send to the CPI via the `create_vm` RPC interface in the networks section. #### Static IP Clarifications -* If single static IPs are defined in the cloud config, the Director verifies that these IPs are base addresses of the specified prefix. If not, an error is raised. -* If a range of static IP addresses is defined, only the base addresses of the specified prefix are considered as static IPs. +- If single static IPs are defined in the cloud config, the Director verifies that these IPs are base addresses of the specified prefix. If not, an error is raised. +- If a range of static IP addresses is defined, only the base addresses of the specified prefix are considered as static IPs. **Considering the following instance group configuration:** @@ -187,16 +189,16 @@ instance_groups: The Director will send two IP addresses to the CPI: -* The next available single address from the `my-network` network configuration. -* The next available prefix delegation from the `my-network-with-prefix` network configuration. +- The next available single address from the `my-network` network configuration. +- The next available prefix delegation from the `my-network-with-prefix` network configuration. #### Limitations -* Networks with a `prefix` defined can only be attached as a secondary network. To group networks to be attached to the same network interface refer to the nic_group configuration in the networks section [here](manifest-v2.md#instance-groups) -* Dynamic and VIP networks are not supported. -* Managed networks are not supported. -* Single static IPs must be a base address of the prefix. -* For IPv6 use cases: Currently, static IP ranges or CIDRs defined on a network where BOSH will assign the next available IP address are extended into an array. Large ranges or CIDRs may lead to performance degradation of the Director. This is particularly relevant for IPv6 addressing, where CIDR ranges easily contain hundreds of millions of addresses. Size `/112` static ranges for networks without prefix delegation seem manageable, at ca. 65k addresses, but at the moment it is recommended to stay below such sizes. +- Networks with a `prefix` defined can only be attached as a secondary network. To group networks to be attached to the same network interface refer to the nic_group configuration in the networks section [here](manifest-v2.md#instance-groups) +- Dynamic and VIP networks are not supported. +- Managed networks are not supported. +- Single static IPs must be a base address of the prefix. +- For IPv6 use cases: Currently, static IP ranges or CIDRs defined on a network where BOSH will assign the next available IP address are extended into an array. Large ranges or CIDRs may lead to performance degradation of the Director. This is particularly relevant for IPv6 addressing, where CIDR ranges easily contain hundreds of millions of addresses. Size `/112` static ranges for networks without prefix delegation seem manageable, at ca. 65k addresses, but at the moment it is recommended to stay below such sizes. See supported CPIs in the [CPI Limitations](#cpi-limitations) section. @@ -212,10 +214,10 @@ Dynamic networking only supports automatic IP reservations. Schema for dynamic network definition: -* **name** [String, required]: Name used to reference this network configuration -* **type** [String, required]: Value should be `dynamic` -* **dns** [Array, optional]: DNS IP addresses for this network -* **cloud_properties** [Hash, optional]: Describes any IaaS-specific properties for the network. Default is `{}` (empty Hash). +- **name** [String, required]: Name used to reference this network configuration +- **type** [String, required]: Value should be `dynamic` +- **dns** [Array, optional]: DNS IP addresses for this network +- **cloud_properties** [Hash, optional]: Describes any IaaS-specific properties for the network. Default is `{}` (empty Hash). Example cloud config: @@ -228,14 +230,13 @@ networks: ``` Schema for dynamic network definition with multiple subnets (available in v241+): - -* **name** [String, required]: Name used to reference this network configuration -* **type** [String, required]: Value should be `dynamic` -* **subnets** [Array, required]: Lists subnets in this network. - * **dns** [Array, optional]: DNS IP addresses for this subnet - * **az** [String, optional]: AZ associated with this subnet (should only be used when using [first class AZs](azs.md)). Example: `z1`. - * **azs** [Array, optional]: List of AZs associated with this subnet (should only be used when using [first class AZs](azs.md)). Example: `[z1, z2]`. - * **cloud_properties** [Hash, optional]: Describes any IaaS-specific properties for the subnet. Default is `{}` (empty Hash). +- **name** [String, required]: Name used to reference this network configuration +- **type** [String, required]: Value should be `dynamic` +- **subnets** [Array, required]: Lists subnets in this network. + - **dns** [Array, optional]: DNS IP addresses for this subnet + - **az** [String, optional]: AZ associated with this subnet (should only be used when using [first class AZs](azs.md)). Example: `z1`. + - **azs** [Array, optional]: List of AZs associated with this subnet (should only be used when using [first class AZs](azs.md)). Example: `[z1, z2]`. + - **cloud_properties** [Hash, optional]: Describes any IaaS-specific properties for the subnet. Default is `{}` (empty Hash). Example cloud config: @@ -260,9 +261,9 @@ VIP network static IPs can either be defined in the deployment manifest (static Schema for VIP network where static IPs are configured in the deployment manifest: -* **name** [String, required]: Name used to reference this network configuration -* **type** [String, required]: Value should be `vip` -* **cloud_properties** [Hash, optional]: Describes any IaaS-specific properties for the network. Default is `{}` (empty Hash). +- **name** [String, required]: Name used to reference this network configuration +- **type** [String, required]: Value should be `vip` +- **cloud_properties** [Hash, optional]: Describes any IaaS-specific properties for the network. Default is `{}` (empty Hash). Sample cloud config and deployment manifest: @@ -290,13 +291,13 @@ instance_groups: Schema for VIP network where static IPs are configured in the cloud config for use across deployments: -* **name** [String, required]: Name used to reference this network configuration -* **type** [String, required]: Value should be `vip` -* **subnets** [Array, optional]: Lists subnets in this network - * **az** [String, optional]: AZ associated with this subnet (should only be used when using [first class AZs](azs.md)). Example: `z1`. - * **azs** [Array, optional]: List of AZs associated with this subnet (should only be used when using [first class AZs](azs.md)). Example: `[z1, z2]`. - * **static** [Array, optional]: Array of static IPs and/or IP ranges. BOSH assigns IPs from this range to instances requesting static IPs. Only IPs specified here can be used for static IP reservations. - * **cloud_properties** [Hash, optional]: Describes any IaaS-specific properties for the subnet. Default is `{}` (empty Hash). +- **name** [String, required]: Name used to reference this network configuration +- **type** [String, required]: Value should be `vip` +- **subnets** [Array, optional]: Lists subnets in this network + - **az** [String, optional]: AZ associated with this subnet (should only be used when using [first class AZs](azs.md)). Example: `z1`. + - **azs** [Array, optional]: List of AZs associated with this subnet (should only be used when using [first class AZs](azs.md)). Example: `[z1, z2]`. + - **static** [Array, optional]: Array of static IPs and/or IP ranges. BOSH assigns IPs from this range to instances requesting static IPs. Only IPs specified here can be used for static IP reservations. + - **cloud_properties** [Hash, optional]: Describes any IaaS-specific properties for the subnet. Default is `{}` (empty Hash). Sample cloud config and deployment manifest: @@ -433,7 +434,7 @@ An instance group can be configured to have multiple IP addresses (multiple NICs Schema for `default` property: -* **default** [Array, optional]: Configures this network to provide its settings for specific category as a default. Possible values are: `dns`, `gateway` and since bosh-release v258 `addressable`. All values can be specified together. `addressable` can be used to specify which IP address other instances see. +- **default** [Array, optional]: Configures this network to provide its settings for specific category as a default. Possible values are: `dns`, `gateway` and since bosh-release v258 `addressable`. All values can be specified together. `addressable` can be used to specify which IP address other instances see. Example: @@ -483,7 +484,7 @@ In the above example, VM allocated to `my-multi-homed-instance-group` instance g The Director does not enforce how many networks can be assigned to each instance; however, each CPI might impose custom requirements either due to the IaaS limitations or simply because support was not yet implemented. | | Manual network
(per instance group) | Dynamic network
(per instance group) | VIP network | -| ------------- | ------------------------------------------ | --------------------------------------- | -------------------- | +|---------------|--------------------------------------------|-----------------------------------------|----------------------| | **AWS** | Multiple1 (from v107.0.0) | Single | Single (Elastic IP) | | **Azure** | Multiple | Multiple | Single (Reserved IP) | | **OpenStack** | [Multiple](openstack-multiple-networks.md) | Single | Single (Floating IP) | @@ -509,10 +510,10 @@ The Director does not enforce how many networks can be assigned to each instance 1: The maximum number of IP addresses assigned to one NIC (limited by the AWS CPI as of now): -* one IPv4 address -* one IPv6 address -* one IPv4 prefix delegation -* one IPv6 prefix delegation +- one IPv4 address +- one IPv6 address +- one IPv4 prefix delegation +- one IPv6 prefix delegation 2: Find the currently supported prefix sizes in [Prefix delegation for AWS](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-prefix-eni.html). diff --git a/content/noble-migration.md b/content/noble-migration.md index 951afffe4..b85e62e8b 100644 --- a/content/noble-migration.md +++ b/content/noble-migration.md @@ -1,3 +1,5 @@ +# Migrating to Noble Numbat + Cloud Foundry's upcoming stemcells will be based on Ubuntu's [Noble Numbat](https://wiki.ubuntu.com/Releases) release, which may cause compilation and deployment errors in packages built for earlier stemcells. This document provides guidance on how to address the most common errors that BOSH release authors may encounter. There are a few broad categories to address: - BOSH DNS — see [below](#bosh-dns) @@ -7,7 +9,7 @@ Cloud Foundry's upcoming stemcells will be based on Ubuntu's [Noble Numbat](http Discussion Slack channel is [here](https://cloudfoundry.slack.com/archives/C06HTDT78N9). -### BOSH DNS +## BOSH DNS In Noble we switched from resolved to systemd-resolve, with this change and to be backwards compatible with our bosh-dns release some configuration are necessary in the runtime config for DNS. If you are using the latest bosh-deployment or bosh bootloader, then you can ignore this. @@ -42,17 +44,18 @@ we added the following configuration. name: bosh-dns-systemd ``` -### BPM +## BPM Use BPM version v1.4.0 or higher [bpm-releases](https://github.com/cloudfoundry/bpm-release/releases) -### EFI Bootloader +## EFI Bootloader The Noble stemcells will use by default the EFI bootloader with a fallback to the legacy bootloader. What this will mean in a real life example for AWS. The vm type `m4.large` (which is deprecated) only supports legacy bootloader you can see what the vm type support with the following command `aws ec2 describe-instance-types --region us-east-1 --instance-types m4.large --query "InstanceTypes[*].SupportedBootModes"` this will result in + ```json [ [ @@ -60,7 +63,9 @@ this will result in ] ] ``` + and for `m5.large` + ```json [ [ @@ -73,8 +78,7 @@ and for `m5.large` This will mean when you use the `m4.large` it will boot in legacy bootloader and for `m5.large` you will boot with the efi bootloader. You can easily check with which bootloader you started by checking if the following file exists `ls /sys/firmware/efi` if this file exists you are in EFI mode and if not you are using the legacy bootloader. - -### Addons (Runtime Configurations) +## Addons (Runtime Configurations) If you restrict your addons to certain stemcells, be sure to include Noble in your list of stemcells (if you intend your addon to run on Noble). The following is the updated stemcell list for [cf-deployment](https://github.com/cloudfoundry/cf-deployment)'s manifest: diff --git a/content/ntp-config.md b/content/ntp-config.md index 0b27767dd..8152535a0 100644 --- a/content/ntp-config.md +++ b/content/ntp-config.md @@ -12,7 +12,7 @@ To configure NTP servers for all VMs on all deployments, add an ntp section to the agent env in your bosh director manifest. For example: -``` +```yaml instance_groups - name: bosh ... @@ -38,7 +38,7 @@ ntp should be configured by adding an ntp section to the `resource_pools` in the create-env manifest For example: -``` +```yaml resource_pools: - env: bosh: @@ -53,7 +53,6 @@ resource_pools: This will have the same agent behavior as changing the `agent.env.bosh.ntp` in a director manifest. - At this point you should know enough to configure NTP on your VMs. The rest of this document is explaining details or exceptions. ## It is also possible to configure the agent env on an instance-group basis diff --git a/content/openstack-cpi-errors.md b/content/openstack-cpi-errors.md index 5aedb68b1..3b4e5ff14 100644 --- a/content/openstack-cpi-errors.md +++ b/content/openstack-cpi-errors.md @@ -1,27 +1,33 @@ +# OpenStack CPI Common Errors + ## Invalid Private Key File -> Command 'deploy' failed: -> Deploying: -> Creating instance 'bosh/0': -> Waiting until instance is ready: -> Starting SSH tunnel: -> Parsing private key file './bosh.pem': -> asn1: structure error: superfluous leading zeros in length +```shell + Command 'deploy' failed: + Deploying: + Creating instance 'bosh/0': + Waiting until instance is ready: + Starting SSH tunnel: + Parsing private key file './bosh.pem': + asn1: structure error: superfluous leading zeros in length +``` If you're using OpenStack Liberty or Mitaka, you cannot use SSH keys generated by nova with BOSH CLI [due to an OpenStack bug](https://bugs.launchpad.net/nova/+bug/1483132). OpenStack versions before Liberty and after Mitaka are not affected. As a workaround, [generate your ssh key manually](https://help.github.com/articles/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent/#generating-a-new-ssh-key) and import it to nova. - ## VM Creation Failed -> Bosh::Clouds::VMCreationFailed +```shell +Bosh::Clouds::VMCreationFailed +``` This error is raised if OpenStack is unable to create a VM. That may happen because: - not enough resources (vCPUs, RAM, disk) to run the VM. For example if you have selected `m1.xlarge` flavor that uses 10 vCPUs and you have 4 hypervisors and each one of them only has 3 vCPUs available, OpenStack is unable to start the VM anywhere even though, total vCPUs across all hypervisors is more than enough. - ## Image Not Found -> Image `4c1d6840-6ac7-4b42-bf29-c95fef6d986e' not found +```shell +Image `4c1d6840-6ac7-4b42-bf29-c95fef6d986e' not found +``` It's possible that image was deleted from OpenStack directly and BOSH is not aware of it. You can recover with `bosh upload stemcell X --fix` to reupload the stemcell. diff --git a/content/openstack-cpi.md b/content/openstack-cpi.md index e3e8faf99..12ffa5e8c 100644 --- a/content/openstack-cpi.md +++ b/content/openstack-cpi.md @@ -1,10 +1,12 @@ +# OpenStack CPI Usage + This topic describes cloud properties for different resources created by the OpenStack CPI. ## AZs {: #azs } Schema for `cloud_properties` section: -* **availability_zone** [String, required]: Availability zone to use for creating instances. Example: `east`. +- **availability_zone** [String, required]: Availability zone to use for creating instances. Example: `east`. Example: @@ -16,12 +18,13 @@ azs: ``` --- + ## Networks {: #networks } Schema for `cloud_properties` section used by dynamic network or manual network subnet: -* **net_id** [String, required]: Network ID containing the subnet in which the instance will be created. Example: `net-b98ab66e-6fae-4c6a-81af-566e630d21d1`. -* **security_groups** [Array, optional]: Array of security group names or UUIDs to apply for all VMs that are placed on this network. Defaults to security groups specified by `default_security_groups` in the global CPI settings unless security groups are specified on a resource pool/vm type for a VM. If security groups are specified on a resource pool and a network, the resource pool security groups takes precedence since CPI v34+. In older CPI versions prior v34, security groups can either be specified for a network or a resource pool. Security group UUIDs can be used since CPI v39+. +- **net_id** [String, required]: Network ID containing the subnet in which the instance will be created. Example: `net-b98ab66e-6fae-4c6a-81af-566e630d21d1`. +- **security_groups** [Array, optional]: Array of security group names or UUIDs to apply for all VMs that are placed on this network. Defaults to security groups specified by `default_security_groups` in the global CPI settings unless security groups are specified on a resource pool/vm type for a VM. If security groups are specified on a resource pool and a network, the resource pool security groups takes precedence since CPI v34+. In older CPI versions prior v34, security groups can either be specified for a network or a resource pool. Security group UUIDs can be used since CPI v39+. Example of manual network: @@ -56,22 +59,23 @@ networks: ``` --- + ## VM Types / VM Extensions {: #resource-pools } Schema for `cloud_properties` section: -* **instance_type** [String, required]: Type of the instance. Example: `m1.small`. -* **availability_zone** [String, required]: Availability zone to use for creating instances. Example: `east`. -* **security_groups** [Array, optional]: Array of security group names or UUIDs to apply for all VMs that are placed on this network. Defaults to security groups specified by `default_security_groups` in the global CPI settings unless security groups are specified on one of the VM networks. If security groups are specified on a resource pool and a network, the resource pool security groups takes precedence since CPI v34+. In older CPI versions prior v34, security groups can either be specified for a network or a resource pool. Security group UUIDs can be used since CPI v39+. -* **key_name** [String, optional]: Key pair name. Defaults to key pair name specified by `default_key_name` in the global CPI settings. Example: `bosh`. -* **scheduler_hints** [Hash, optional]: Data passed to the OpenStack Filter scheduler to influence its decision where new VMs can be placed. See [VM Anti-Affinity](vm-anti-affinity.md#openstack) for a detailed example. Example: `{ group: af09abf2-2283... }` -* **root_disk** [Hash, optional]: Custom root disk properties. Requires `boot_from_volume: true` either [globally](https://bosh.io/jobs/openstack_cpi?source=github.com/cloudfoundry/bosh-openstack-cpi-release#p=openstack.boot_from_volume) or locally in this VM Type to enable cinder-backed boot volumes. Available in v25+. - * **size** [Integer, required]: Specifies the disk size in gigabytes. -* **loadbalancer_pools** [Array, optional]: Array of Hashes defining LBaaSv2 pools to attach this instance to. Requires neutron LBaaSv2 extension and OpenStack Mitaka or newer. Available in v32+. - * **name** [String, required]: The name of the LBaaSv2 loadbalancer pool - * **port** [Integer, required]: The port exposed on the instance -* **boot\_from\_volume** [Boolean, optional]: Override global [`boot_from_volume`](https://bosh.io/jobs/openstack_cpi?source=github.com/cloudfoundry/bosh-openstack-cpi-release#p=openstack.boot_from_volume) to enable cinder-backed boot volumes for this VM Type. Available in v34+. -* **allowed_address_pairs** [String, optional]: A single VRRP IP address that associated VMs will be allowed to use in addition to their primary IP address. See [Using VRRP](openstack-vrrp.md) for more details. Available in v37+. +- **instance_type** [String, required]: Type of the instance. Example: `m1.small`. +- **availability_zone** [String, required]: Availability zone to use for creating instances. Example: `east`. +- **security_groups** [Array, optional]: Array of security group names or UUIDs to apply for all VMs that are placed on this network. Defaults to security groups specified by `default_security_groups` in the global CPI settings unless security groups are specified on one of the VM networks. If security groups are specified on a resource pool and a network, the resource pool security groups takes precedence since CPI v34+. In older CPI versions prior v34, security groups can either be specified for a network or a resource pool. Security group UUIDs can be used since CPI v39+. +- **key_name** [String, optional]: Key pair name. Defaults to key pair name specified by `default_key_name` in the global CPI settings. Example: `bosh`. +- **scheduler_hints** [Hash, optional]: Data passed to the OpenStack Filter scheduler to influence its decision where new VMs can be placed. See [VM Anti-Affinity](vm-anti-affinity.md#openstack) for a detailed example. Example: `{ group: af09abf2-2283... }` +- **root_disk** [Hash, optional]: Custom root disk properties. Requires `boot_from_volume: true` either [globally](https://bosh.io/jobs/openstack_cpi?source=github.com/cloudfoundry/bosh-openstack-cpi-release#p=openstack.boot_from_volume) or locally in this VM Type to enable cinder-backed boot volumes. Available in v25+. + - **size** [Integer, required]: Specifies the disk size in gigabytes. +- **loadbalancer_pools** [Array, optional]: Array of Hashes defining LBaaSv2 pools to attach this instance to. Requires neutron LBaaSv2 extension and OpenStack Mitaka or newer. Available in v32+. + - **name** [String, required]: The name of the LBaaSv2 loadbalancer pool + - **port** [Integer, required]: The port exposed on the instance +- **boot\_from\_volume** [Boolean, optional]: Override global [`boot_from_volume`](https://bosh.io/jobs/openstack_cpi?source=github.com/cloudfoundry/bosh-openstack-cpi-release#p=openstack.boot_from_volume) to enable cinder-backed boot volumes for this VM Type. Available in v34+. +- **allowed_address_pairs** [String, optional]: A single VRRP IP address that associated VMs will be allowed to use in addition to their primary IP address. See [Using VRRP](openstack-vrrp.md) for more details. Available in v37+. Example of an `m1.small` instance: @@ -122,11 +126,12 @@ resource_pools: ``` --- + ## Disk Types {: #disk-pools } Schema for `cloud_properties` section: -* **type** [String, optional]: Volume type as configured in your OpenStack installation. Example: `SSD` +- **type** [String, optional]: Volume type as configured in your OpenStack installation. Example: `SSD` Cinder volumes are created in the availability zone of an instance that volume will be attached. @@ -140,13 +145,14 @@ Example of 10GB disk: ``` --- + ## Global Configuration {: #global } See [CPI job configuration](https://bosh.io/jobs/openstack_cpi?source=github.com/cloudfoundry/bosh-openstack-cpi-release) for details. Schema: -* **default_volume_type** [String, optional]: sets volume type for persistent disks unless overridden in resource pool/VM Type. `cinder type-list` will return the available volume types. Example: `SSD`. +- **default_volume_type** [String, optional]: sets volume type for persistent disks unless overridden in resource pool/VM Type. `cinder type-list` will return the available volume types. Example: `SSD`. Example with Keystone V3 and a single domain: @@ -160,6 +166,7 @@ region: RegionOne default_key_name: bosh default_security_groups: [bosh] ``` + Example with Keystone V3 and different domains for the user and project: ```yaml @@ -188,6 +195,7 @@ default_volume_type: ceph ``` --- + ## Example Cloud Config {: #cloud-config } ```yaml diff --git a/content/openstack-human-readable-vm-names.md b/content/openstack-human-readable-vm-names.md index c3028a276..8da03e234 100644 --- a/content/openstack-human-readable-vm-names.md +++ b/content/openstack-human-readable-vm-names.md @@ -1,3 +1,5 @@ +# OpenStack - Using Human-readable VM names + !!! note This feature is available with bosh-openstack-cpi v23+. diff --git a/content/openstack-keystonev2.md b/content/openstack-keystonev2.md index 5ffe9d288..b9bff12a6 100644 --- a/content/openstack-keystonev2.md +++ b/content/openstack-keystonev2.md @@ -1,3 +1,5 @@ +# Using Keystone API v2 + Default configuration typically uses Keystone v3 API. This document describes how to use Keystone v2 if your OpenStack installation enforces this. 1. Configure OpenStack CPI diff --git a/content/openstack-light-stemcells.md b/content/openstack-light-stemcells.md index 23f54e13c..76163b66d 100644 --- a/content/openstack-light-stemcells.md +++ b/content/openstack-light-stemcells.md @@ -1,16 +1,19 @@ +# Using Light Stemcells + !!! note This feature is available with bosh-openstack-cpi v28+. !!! note This feature is available with bosh-cli v2.0.40+. -You can create your own OpenStack light stemcells to re-use stemcell images already uploaded to your OpenStack image store. +You can create your own OpenStack light stemcells to re-use stemcell images already uploaded to your OpenStack image store. **Note:** Future deployments will fail if the stemcell image referenced by a light stemcell is removed from your OpenStack image store. 1. Download the heavy stemcell for which you want to create a light stemcell 1. Upload the stemcell to your OpenStack with `bosh upload-stemcell` and retrieve the CID and version with `bosh stemcells`. In case this is not possible, please read the example [Manually upload stemcell with OpenStack CLI](#example-manually-upload-stemcell-with-openstack-cli) 1. Use `bosh repack-stemcell` to create a light stemcell archive from a heavy stemcell + ```shell bosh repack-stemcell --version "" \ --empty-image \ @@ -21,10 +24,7 @@ heavy-stemcell.tgz ./light-bosh-stemcell--openstack-kvm-ubuntu You can use the light stemcell archive like a regular stemcell archive in BOSH deployment manifests and with `bosh create-env` command. - - -###Example: Manually upload stemcell with OpenStack CLI - +## Example: Manually upload stemcell with OpenStack CLI Untar the downloaded heavy stemcell and its `image` to extract the `root.img` diff --git a/content/openstack-multiple-networks.md b/content/openstack-multiple-networks.md index 0a58b3385..6a978608f 100644 --- a/content/openstack-multiple-networks.md +++ b/content/openstack-multiple-networks.md @@ -1,21 +1,21 @@ +# OpenStack Multi-homed VMs + !!! note This feature is available with bosh-openstack-cpi v24+. !!! note This feature requires OpenStack Neutron. -### Limitation: This feature requires DHCP to be disabled +## Limitation: This feature requires DHCP to be disabled Disabling DHCP means that the network devices on your VMs will not get configuration such as default gateway, DNS, and MTU. If you require specific values for these settings, you will need to set them by other means. -1. In your Director deployment manifest, set [`properties.openstack.use_dhcp: false`] - (https://bosh.io/jobs/openstack_cpi?source=github.com/cloudfoundry/bosh-openstack-cpi-release#p=openstack.use_dhcp). +1. In your Director deployment manifest, set [`properties.openstack.use_dhcp: false`](https://bosh.io/jobs/openstack_cpi?source=github.com/cloudfoundry/bosh-openstack-cpi-release#p=openstack.use_dhcp). This means the BOSH agent will configure the network devices without DHCP. This is a Director-wide setting and switches off DHCP for all VMs deployed with this Director. -1. In your Director deployment manifest, set [`properties.openstack.config_drive: cdrom`] - (https://bosh.io/jobs/openstack_cpi?source=github.com/cloudfoundry/bosh-openstack-cpi-release#p=openstack.config_drive). +1. In your Director deployment manifest, set [`properties.openstack.config_drive: cdrom`](https://bosh.io/jobs/openstack_cpi?source=github.com/cloudfoundry/bosh-openstack-cpi-release#p=openstack.config_drive). This means OpenStack will mount a cdrom drive to distribute meta-data and user-data instead of using an HTTP metadata service. 1. In your [BOSH network configuration](networks.md#manual), set `gateway` and `dns` to allow outbound communication. 1. If you're not using VLAN, but a tunnel mechanism for Neutron networking, you also need to set the MTU for your network devices on *all* VMs: diff --git a/content/openstack-registry.md b/content/openstack-registry.md index 78d3293fb..56bb993b0 100644 --- a/content/openstack-registry.md +++ b/content/openstack-registry.md @@ -1,3 +1,5 @@ +# OpenStack Extended Registry Configuration + !!! note We are actively pursuing to remove the Registry to simplify BOSH architecture. diff --git a/content/openstack-self-signed-endpoints.md b/content/openstack-self-signed-endpoints.md index 740f11416..051be3a18 100644 --- a/content/openstack-self-signed-endpoints.md +++ b/content/openstack-self-signed-endpoints.md @@ -1,3 +1,5 @@ +# OpenStack Self-Signed Endpoints + !!! note This feature is available with bosh-openstack-cpi v23+. diff --git a/content/openstack-vrrp.md b/content/openstack-vrrp.md index bf7a12923..e9539a67a 100644 --- a/content/openstack-vrrp.md +++ b/content/openstack-vrrp.md @@ -1,3 +1,5 @@ +# OpenStack - Using VRRP + !!! note This feature is available with bosh-openstack-cpi v37+. @@ -7,30 +9,36 @@ In order for this to work, your OpenStack network needs to support VRRP and mult As the OpenStack CPI takes care of creating neutron ports attached to bosh vms for you, you cannot set this property yourself. Instead, you tell the OpenStack CPI to set `allowed_address_pairs` automatically the ports by using a `vm_extension`: -* create a neutron port with the VRRP IP you want to expose to your clients. We will not attach this port to any VM, it's rather a way to "reserve" the IP address that will later be allowed on the bosh VM ports. Following is a sample terraform script for automating such port creation: -``` - resource "openstack_networking_port_v2" "vrrp_port" { - name = "my_cluster_virtual_ip_port" - network_id = openstack_networking_network_v2.my_net.id - fixed_ip { +- create a neutron port with the VRRP IP you want to expose to your clients. We will not attach this port to any VM, it's rather a way to "reserve" the IP address that will later be allowed on the bosh VM ports. Following is a sample terraform script for automating such port creation: + +```terraform + resource "openstack_networking_port_v2" "vrrp_port" { + name = "my_cluster_virtual_ip_port" + network_id = openstack_networking_network_v2.my_net.id + fixed_ip { subnet_id = openstack_networking_subnet_v2.my_subnet.id - ip_address = "x.x.x.x" #The VRRP IP - } - admin_state_up = "true" -``` -* create a `vm_extension` in your `cloud-config`: + ip_address = "x.x.x.x" #The VRRP IP + } + admin_state_up = "true" ``` + +- create a `vm_extension` in your `cloud-config`: + +```yaml vm_extensions: - name: vrrp-ip cloud_properties: allowed_address_pairs: ``` -* Use the `vm_extension` in your deployment manifest like this, to select the instance groups on which it will apply. To ensure consistency and fail fast, the OpenStack CPI will check the presence of the OpenStack port matching the VRRP IP. Also, all vms in this instance group will have their ports configured with the `allowed_address_pairs` property set to the VRRP IP and their mac address, asking OpenStack to allow the VM to send/receive traffic on this IP address. -``` + +- Use the `vm_extension` in your deployment manifest like this, to select the instance groups on which it will apply. To ensure consistency and fail fast, the OpenStack CPI will check the presence of the OpenStack port matching the VRRP IP. Also, all vms in this instance group will have their ports configured with the `allowed_address_pairs` property set to the VRRP IP and their mac address, asking OpenStack to allow the VM to send/receive traffic on this IP address. + +```yaml instance_groups: - name: my-instance-group vm_extensions: [vrrp-ip] ``` -* Co-locate the `keepalived` job of the `haproxy-boshrelease` and configure the VRRP IP as [`keepalived.ip`](https://bosh.io/jobs/keepalived?source=github.com/cloudfoundry-community/haproxy-boshrelease#p%3dkeepalived.vip) + +- Co-locate the `keepalived` job of the `haproxy-boshrelease` and configure the VRRP IP as [`keepalived.ip`](https://bosh.io/jobs/keepalived?source=github.com/cloudfoundry-community/haproxy-boshrelease#p%3dkeepalived.vip) When your master vm goes down, the VRRP IP will be attached to your slave vm and all clients don't need to be updated, they keep communicating to the VRRP IP. diff --git a/content/openstack.md b/content/openstack.md index 0b788e233..3c2bcadde 100644 --- a/content/openstack.md +++ b/content/openstack.md @@ -1,15 +1,10 @@ ---- -title: OpenStack ---- - # OpenStack The `openstack` CPI can be used with [OpenStack](https://www.openstack.org). - * Release: [cloudfoundry/bosh-openstack-cpi-release](https://github.com/cloudfoundry/bosh-openstack-cpi-release) - * Issues: [GitHub Issues](https://github.com/cloudfoundry/bosh-openstack-cpi-release/issues) - * Slack: [cloudfoundry#openstack](https://cloudfoundry.slack.com/messages/openstack) - +- Release: [cloudfoundry/bosh-openstack-cpi-release](https://github.com/cloudfoundry/bosh-openstack-cpi-release) +- Issues: [GitHub Issues](https://github.com/cloudfoundry/bosh-openstack-cpi-release/issues) +- Slack: [cloudfoundry#openstack](https://cloudfoundry.slack.com/messages/openstack) ## Requirements @@ -17,34 +12,33 @@ An OpenStack environment that is [supported by the CPI](https://github.com/cloud And the following OpenStack services: - * [Identity](https://www.openstack.org/software/releases/latest/components/keystone): - BOSH authenticates credentials and retrieves the endpoint URLs for other OpenStack services. - * [Compute](https://www.openstack.org/software/releases/latest/components/nova): - BOSH boots new VMs, assigns floating IPs to VMs - * [Image](https://www.openstack.org/software/releases/latest/components/glance): - BOSH stores stemcells using the Image service. As of v40 of the OpenStack CPI, v2 of the Image service is required. - * *(Optional)* [OpenStack Networking](https://www.openstack.org/software/releases/latest/components/neutron): - Provides network scaling and automated management functions that are useful when deploying complex distributed systems. **Note:** OpenStack networking is used as default as of v28 of the OpenStack CPI. - * *(Optional)* [OpenStack Block Storage](https://www.openstack.org/software/releases/latest/components/cinder): - BOSH creates persistent volumes. While it is technically possible to use BOSH on OpenStack without block storage, you won't get persistent volumes without it. As of v40 of the OpenStack CPI, v2 of the Block Storage service is required. +- [Identity](https://www.openstack.org/software/releases/latest/components/keystone): + BOSH authenticates credentials and retrieves the endpoint URLs for other OpenStack services. +- [Compute](https://www.openstack.org/software/releases/latest/components/nova): + BOSH boots new VMs, assigns floating IPs to VMs +- [Image](https://www.openstack.org/software/releases/latest/components/glance): + BOSH stores stemcells using the Image service. As of v40 of the OpenStack CPI, v2 of the Image service is required. +- *(Optional)* [OpenStack Networking](https://www.openstack.org/software/releases/latest/components/neutron): + Provides network scaling and automated management functions that are useful when deploying complex distributed systems. **Note:** OpenStack networking is used as default as of v28 of the OpenStack CPI. +- *(Optional)* [OpenStack Block Storage](https://www.openstack.org/software/releases/latest/components/cinder): + BOSH creates persistent volumes. While it is technically possible to use BOSH on OpenStack without block storage, you won't get persistent volumes without it. As of v40 of the OpenStack CPI, v2 of the Block Storage service is required. ## Concepts The following table maps BOSH concepts to their OpenStack-native equivalents. -| BOSH | OpenStack | -| ----------------- | --------- | -| Availability Zone | [Availability Zone](https://www.mirantis.com/blog/the-first-and-final-word-on-openstack-availability-zones/) | -| Virtual Machine | [Instance](https://docs.openstack.org/nova/latest/user/launch-instances.html) | -| Instance Type | [Flavor](https://docs.openstack.org/nova/latest/user/flavors.html) | -| Network Subnet | [Subnet](https://docs.openstack.org/neutron/latest/admin/intro-os-networking.html) | -| Virtual IP | [Floating IP](https://docs.openstack.org/nova/latest/user/manage-ip-addresses.html) | -| Persistent Disk | [Volume](https://docs.openstack.org/cinder/latest/cli/cli-manage-volumes.html) | -| Disk Snapshot | [Volume Snapshot](https://docs.openstack.org/cinder/latest/cli/cli-manage-volumes.html) | -| Stemcell | [Virtual Machine Image](https://docs.openstack.org/glance/latest/user/index.html) | +| BOSH | OpenStack | +|-------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Availability Zone | [Availability Zone](https://www.mirantis.com/blog/the-first-and-final-word-on-openstack-availability-zones/) | +| Virtual Machine | [Instance](https://docs.openstack.org/nova/latest/user/launch-instances.html) | +| Instance Type | [Flavor](https://docs.openstack.org/nova/latest/user/flavors.html) | +| Network Subnet | [Subnet](https://docs.openstack.org/neutron/latest/admin/intro-os-networking.html) | +| Virtual IP | [Floating IP](https://docs.openstack.org/nova/latest/user/manage-ip-addresses.html) | +| Persistent Disk | [Volume](https://docs.openstack.org/cinder/latest/cli/cli-manage-volumes.html) | +| Disk Snapshot | [Volume Snapshot](https://docs.openstack.org/cinder/latest/cli/cli-manage-volumes.html) | +| Stemcell | [Virtual Machine Image](https://docs.openstack.org/glance/latest/user/index.html) | | Agent Settings | [HTTP Metadata service](https://docs.openstack.org/nova/latest/user/metadata-service.html); [Config Drive](https://docs.openstack.org/nova/latest/user/config-drive.html); BOSH Registry | - ## Feature Support The following sections describe some specific BOSH features supported by the diff --git a/content/package-vendoring.md b/content/package-vendoring.md index 18c2ef4e7..0176c8880 100644 --- a/content/package-vendoring.md +++ b/content/package-vendoring.md @@ -1,3 +1,5 @@ +# Reusing Packages + !!! note Requires CLI v2.0.36+. @@ -22,8 +24,9 @@ CLI v2 introduces new command for release authors to easily vendor final version As an example, if release encapsulates a Go application that needs to be compiled with Go compiler (as most Go apps do), release author, instead of figuring out how to make a `golang-1.x` package on their own, can vendor in one from `https://github.com/cloudfoundry/bosh-package-golang-release`. ### Vendoring by example + Here an example how package vendoring could work from scratch. A local blobstore is used for simplicity. -I a productive scenario may use blobstores like Amazon S3 as documented [here](release-blobstore.md). +In a productive scenario, you may use blobstores like Amazon S3 as documented in [Release Blobstore](release-blobstore.md). ```shell # Create a release skeleton @@ -44,7 +47,8 @@ blobstore: bosh vendor-package golang-1.18-linux ~/workspace/bosh-package-golang-release ``` -After running the `vendor-package` command +After running the `vendor-package` command: + - The local blobstore in `/tmp/local-blobstore` contains the vendored package - The uploaded package is referenced in `.final_builds/packages/golang-1.18-linux/index.yml` - A new package `golang-1.18-linux` is added. That package references the vendored package in `.final_builds` by the `spec.lock` file. @@ -52,14 +56,17 @@ After running the `vendor-package` command The `spec.lock` file and the updates to `.final_builds` have to be checked in. During development be aware of caching: + - The existence of `.final_builds/packages/golang-1.18-linux/index.yml` prohibits further uploads of the package to the local blobstore. - Downloaded releases are cached in `~/.bosh/cache` ### Referencing vendored package + In the above steps, CLI v2 vendors the `golang-1.18-linux` package into your `my-app-release` release. Other packages of `my-app-release` could now reference `golang-1.18-linux` as a dependency just like any other package in the spec file. Here an example: + ```yaml name: a-depending-package @@ -87,7 +94,7 @@ In the above BASH script, `source /var/vcap/packages/golang-1.18-linux/bosh/comp Packages may also include `bosh/runtime.env` for loading specific functionality at job runtime instead of during package compilation. -### Additional notes about `vendor-package` command: +### Additional notes about `vendor-package` command - The command is idempotent, hence could be run in the CI continuously tracking source release and automatically vendoring in updates. - The command requires [access to the final blobstore](release-blobstore.md) as it will download the source release package blob and upload it into destination release's blobstore. @@ -104,6 +111,7 @@ When to be cautious with this approach: - package's purpose or implementation is extremely specific to the source release --- + ## Using job colocation {: #colocation } Job colocation can provide a powerful way to make a release extensible and pluggable where necessary. Unlike vendoring approach, release author choosing job colocation as a way to consume dependent software is explicitly stating that there is not necessarily a single one implementation of a particular dependency but rather it could be chosen by an operator at the time of a deploy. @@ -124,6 +132,7 @@ When to be cautious with this approach: - if operators will incur unnecessary burden during a deploy --- + ## Copying over package source {: #copy } Lastly, sometimes it may be necessary to actually copy over (`cp`) software bits from one release to another. diff --git a/content/packages.md b/content/packages.md index 2bd24047b..4c8c788d7 100644 --- a/content/packages.md +++ b/content/packages.md @@ -1,3 +1,5 @@ +# Creating Packages + A package is a component of a BOSH release that contains a packaging `spec` file and a packaging script. Each package also references source code or pre-compiled software that you store in the `src` directory of a BOSH [release directory](create-release.md). @@ -17,11 +19,11 @@ a BOSH Release topic. You specify package contents in the package `spec` file. BOSH automatically creates this file as a template with the following sections when you run the `bosh generate-package PACKAGE_NAME` command: - * `name`: Defines the package name. - * `dependencies`: **(Optional)** Defines a list of other packages that this package depends on. - * `files`: Defines a list of files that this package contains. You can define this list explicitly or through pattern-matching. - * `excluded_files`: **(Optional)** Defines a list of files to be excluded from the package. You can define this list explicitly or through pattern-matching. - * `no_compression`: **(Optional)** Defines whether compression is disabled for the individual package tarball. Defaults to `false` (compression enabled) if not specified. +- `name`: Defines the package name. +- `dependencies`: **(Optional)** Defines a list of other packages that this package depends on. +- `files`: Defines a list of files that this package contains. You can define this list explicitly or through pattern-matching. +- `excluded_files`: **(Optional)** Defines a list of files to be excluded from the package. You can define this list explicitly or through pattern-matching. +- `no_compression`: **(Optional)** Defines whether compression is disabled for the individual package tarball. Defaults to `false` (compression enabled) if not specified. !!! note "Version Requirements" The `no_compression` flag requires BOSH Director version `282.1.5` or newer and the following stemcell versions: @@ -32,7 +34,7 @@ To edit a package spec file: 1. Identify all compile-time dependencies. A compile-time dependency occurs when a package depends on another package. - For more information, refer to the [Make Dependency Graphs](create-release.md#graph) section of the Creating a BOSH + For more information, refer to the [Make Dependency Graphs](create-release.md#graph) section of the Creating a BOSH Release topic. 1. Run `bosh generate-package PACKAGE_NAME` for each compile-time dependency. 1. Copy all files that the package requires to the `src` directory of the BOSH release directory. @@ -41,8 +43,8 @@ Release topic. pre-compiled binary. 1. Edit each package spec file as follows: - * Add the names of the files for that package. - * Add the names of any compile-time dependencies to each package spec file. Use `[]` to indicate an empty array if a package + - Add the names of the files for that package. + - Add the names of any compile-time dependencies to each package spec file. Use `[]` to indicate an empty array if a package has no compile-time dependencies. The example shows a Ruby spec file with dependencies and file names. @@ -70,7 +72,6 @@ no_compression: true When `no_compression` is set to `true`, the individual package tarball will not be compressed. By default, if `no_compression` is not specified or set to `false`, packages are compressed. - ## Create a Packaging Script {: #create-a-packaging-script } BOSH automatically creates a packaging script file template when you run the `bosh generate-package PACKAGE_NAME` command. Each diff --git a/content/persistent-disk-fs.md b/content/persistent-disk-fs.md index 03c85355e..b1d06ac5d 100644 --- a/content/persistent-disk-fs.md +++ b/content/persistent-disk-fs.md @@ -1,3 +1,5 @@ +# Persistent Disk - Using XFS + !!! note This feature is available with 3215+ stemcell series. diff --git a/content/persistent-disks.md b/content/persistent-disks.md index c53f702ab..863e7cbdd 100644 --- a/content/persistent-disks.md +++ b/content/persistent-disks.md @@ -1,3 +1,5 @@ +# Persistent Disks + !!! note This document was updated to mention orphaned disks introduced in bosh-release v241+ (1.3163.0). @@ -11,27 +13,28 @@ persistent disk data remains intact. Attaching the persistent disk to another VM Persistent disks are kept for each instance under the following circumstances: -* updating deployment to use new releases or stemcells -* using cloud check to recover deleted VMs -* instances are hard stopped and later started again +- updating deployment to use new releases or stemcells +- using cloud check to recover deleted VMs +- instances are hard stopped and later started again As of bosh-release v241+ (1.3163.0), the Director no longer deletes persistent disks that are no longer needed. Unnecessary persistent disks will be marked as orphaned so that they can be garbage collected after 5 days. The following conditions result in persistent disks to be marked as orphaned: -* instance group no longer specifies a persistent disk size or a disk pool -* instance group changes the size or cloud properties of a disk -* instance group is renamed without `migrated_from` configuration -* instance group is scaled down -* instance group is deleted or AZ assignment is removed -* deployment is deleted +- instance group no longer specifies a persistent disk size or a disk pool +- instance group changes the size or cloud properties of a disk +- instance group is renamed without `migrated_from` configuration +- instance group is scaled down +- instance group is deleted or AZ assignment is removed +- deployment is deleted You can specify that an instance group needs an attached persistent disk in one of two ways: -* [Persistent Disk declaration](#persistent-disk) -* [Persistent Disk Pool declaration](#persistent-disk-pool) +- [Persistent Disk declaration](#persistent-disk) +- [Persistent Disk Pool declaration](#persistent-disk-pool) --- + ## Persistent Disk Declaration {: #persistent-disk } To specify that an instance group needs an attached persistent disk, add a `persistent_disk` key-value pair to the instance group in the [Jobs](deployment-manifest.md#jobs) block of your deployment manifest. @@ -56,19 +59,20 @@ instance_groups: If you use persistent disk declaration, you cannot specify the persistent disk type that the CPI attaches to your job VMs. Instead, the CPI uses its default disk configuration when deploying the VMs. --- + ## Persistent Disk Pool Declaration {: #persistent-disk-pool } To specify that an instance group needs an attached persistent disk, add a [Disk Pool](deployment-manifest.md#disk-pools) block to your deployment manifest. The persistent disk pool declaration allows you to specify the precise type and size of the persistent disks attached to your instance group VMs. -* **persistent\_disk_pool** [String, optional]: Associated with an instance group; specifies a particular disk_pool. +- **persistent\_disk_pool** [String, optional]: Associated with an instance group; specifies a particular disk_pool. -* **disk_pools** [Array, optional]: Specifies the [disk_pools](terminology.md#disk-pool) a deployment uses. A deployment manifest can describe multiple disk pools and uses unique names to identify and reference them. +- **disk_pools** [Array, optional]: Specifies the [disk_pools](terminology.md#disk-pool) a deployment uses. A deployment manifest can describe multiple disk pools and uses unique names to identify and reference them. - * **name** [String, required]: A unique name used to identify and reference the disk pool. - * **disk_size** [Integer, required]: Size of the disk in megabytes. - * **cloud_properties** [Hash, optional]: Describes any IaaS-specific properties needed to create disk. Examples: `type`, `ops` + - **name** [String, required]: A unique name used to identify and reference the disk pool. + - **disk_size** [Integer, required]: Size of the disk in megabytes. + - **cloud_properties** [Hash, optional]: Describes any IaaS-specific properties needed to create disk. Examples: `type`, `ops` Example: @@ -94,11 +98,13 @@ instance_groups: ``` --- + ## Checking Stats {: #checking-stats } After your deployment completes, run `bosh vms --vitals` from a terminal window to view persistent disk usage percentage values under `Persistent Disk Usage`. --- + ## Accessing Persistent Disks {: #accessing-persistent-disk } The CPI mounts persistent disks `/var/vcap/store` on deployed VMs, and persists any files stored in `/var/vcap/store`. @@ -108,6 +114,7 @@ You specify jobs using the `jobs` key when defining a instance group. By convent For example, a `redis` job creates the following directory: `/var/vcap/store/redis` --- + ## Changing Disk Properties {: #changing-persistent-disk } BOSH allows you to change disk types and sizes by modifying the deployment manifest. As long as the instance group name stays the same, data on existing persistent disks will be migrated onto new persistent disks. Old persistent disks will be marked as orphaned. @@ -118,6 +125,7 @@ During the disk migration from one disk type and size to another, the Director c An IaaS might disallow attaching particular disk types and sizes to certain VM types. Consult your IaaS documentation for more information. --- + ## Orphaned Disks {: #orphaned-disks } Orphaned persistent disks are not attached to any VM and are not associated with any deployment. You can list orphaned disks known to the Director via [`bosh disks --orphaned` command](sysadmin-commands.md#disks). If deployment changes were done erroneously and you would like to reattach specific orphaned persistent disk to an instance follow these steps: diff --git a/content/personas.md b/content/personas.md index 413f8f82e..ee1cda714 100644 --- a/content/personas.md +++ b/content/personas.md @@ -1,119 +1,112 @@ -BOSH is used by several different groups of people with different needs and goals. Here are the primary personas and their responsibilities that we typically consider... +# Target Personas +BOSH is used by several different groups of people with different needs and goals. Here are the primary personas and their responsibilities that we typically consider... ## Release Author A release author is interested in how software can be deployed and run with BOSH. They will typically be familiar with both the underlying software which will be deployed, and BOSH conventions around how BOSH supports configuring and installing the software. They will want to understand... - * The target software... - * What it means to run it at scale and in production. - * What settings users need to configure. - * What conventions the software may have and users expect. - * What dependencies the software may need. - * What upgrade workflows are effective. - * BOSH strategies for managing software... - * How to write compilation scripts for the software. - * How to write runtime scripts for running the software. - * How to manage versions of their release. - * How deployment operators expect to deploy their release. +- The target software... + - What it means to run it at scale and in production. + - What settings users need to configure. + - What conventions the software may have and users expect. + - What dependencies the software may need. + - What upgrade workflows are effective. +- BOSH strategies for managing software... + - How to write compilation scripts for the software. + - How to write runtime scripts for running the software. + - How to manage versions of their release. + - How deployment operators expect to deploy their release. Some examples of Release Authors are... - * Diego team managing [cloudfoundry/diego-release](https://github.com/cloudfoundry/diego-release) - this release is dedicated to deploying the Diego component of Cloud Foundry Application Runtime. - * Concourse team managing [concourse/concourse](https://github.com/concourse/concourse) - this release integrates several Concourse-specific components into a set of software which makes sense to deploy. - * Datadog team managing [DataDog/datadog-agent-boshrelease](https://github.com/DataDog/datadog-agent-boshrelease) - this release forwards telemetry and other monitoring information to the Datadog hosted services. - +- Diego team managing [cloudfoundry/diego-release](https://github.com/cloudfoundry/diego-release) - this release is dedicated to deploying the Diego component of Cloud Foundry Application Runtime. +- Concourse team managing [concourse/concourse](https://github.com/concourse/concourse) - this release integrates several Concourse-specific components into a set of software which makes sense to deploy. +- Datadog team managing [DataDog/datadog-agent-boshrelease](https://github.com/DataDog/datadog-agent-boshrelease) - this release forwards telemetry and other monitoring information to the Datadog hosted services. ## Deployment Author A deployment author is interested in making it easier for others to deploy releases in a pre-configured way. They may not be directly responsible for creating releases and defining how they're configured, nor managing the software running in production. They will want to understand... - * How release authors expose services and configuration of their software. - * How deployment operators are interested in running the software or set of services. - * When integrating multiple components... - * Which versions of components are safe to integrate with each other. - * Which feature sets may need to be optional or configured (likely with “ops files”) +- How release authors expose services and configuration of their software. +- How deployment operators are interested in running the software or set of services. +- When integrating multiple components... + - Which versions of components are safe to integrate with each other. + - Which feature sets may need to be optional or configured (likely with “ops files”) Some examples of Deployment Authors are... - * Release Integration team managing [cloudfoundry/cf-deployment](https://github.com/cloudfoundry/cf-deployment) - this deployment brings together numerous releases to manage a set of versions and features which can be used to deploy Cloud Foundry. - * Teams developing [PCF tiles](https://docs.pivotal.io/tiledev/2-0/) (Tile Authors) - a tile integrates some customization options for managing deployments and services with Pivotal's Ops Manager product. - +- Release Integration team managing [cloudfoundry/cf-deployment](https://github.com/cloudfoundry/cf-deployment) - this deployment brings together numerous releases to manage a set of versions and features which can be used to deploy Cloud Foundry. +- Teams developing [PCF tiles](https://docs.pivotal.io/tiledev/2-0/) (Tile Authors) - a tile integrates some customization options for managing deployments and services with Pivotal's Ops Manager product. ## Operator Operators are responsible for the ongoing stability of BOSH resources. There are usually two layers of BOSH which people take responsibility for, although sometimes the responsibility for both roles may be shared within a single team. - ### Deployment Operator A deployment operator is interested in telling the director to make sure it deploys and runs software they need on the cloud. They will want to understand... - * How the release software should be configured; - * What other components are involved to successful operate their software; - * What availability zones and networks are available to them for their deployment. - +- How the release software should be configured; +- What other components are involved to successful operate their software; +- What availability zones and networks are available to them for their deployment. ### Director Operator A director operator is interested in configuring and maintaining the director to ensure deployment operators are able to deploy their services to the cloud. They will want to understand... - * What the general director architecture and components look like. - * How to securely configure director components. - * What clouds are available... - * How to connect and authenticate to clouds with CPIs. - * How networks are connected. - * Authorization and authentication control... - * How teams and users connect to the director. - * How deployments will be accessible to others. - +- What the general director architecture and components look like. +- How to securely configure director components. +- What clouds are available... + - How to connect and authenticate to clouds with CPIs. + - How networks are connected. +- Authorization and authentication control... + - How teams and users connect to the director. + - How deployments will be accessible to others. ### Cloud Operator A cloud operator is responsible for planning and managing the underlying infrastructure where resources are managed. They will want to understand... - * How to plan for resource consumption... - * What compute, memory, disk resources are being used and needed. - * What network assignments should be allocated. - * How to provision access to the environment. - * How to monitor the infrastructure to identify issues which may impact availability of higher layers. +- How to plan for resource consumption... + - What compute, memory, disk resources are being used and needed. + - What network assignments should be allocated. +- How to provision access to the environment. +- How to monitor the infrastructure to identify issues which may impact availability of higher layers. Some examples of Cloud Operators are... - * OpenStack or vSphere environments are often managed by a dedicated infrastructure team. - * Public clouds like Google Cloud Platform or Amazon Web Services may have a team responsible for overseeing general usage and defining policies for teams. - +- OpenStack or vSphere environments are often managed by a dedicated infrastructure team. +- Public clouds like Google Cloud Platform or Amazon Web Services may have a team responsible for overseeing general usage and defining policies for teams. ## Developer A developer is interested in enhancing functionality of one or more components of BOSH. There are several different projects a developer may be interested in. - ### Internal Developer An internal developer is focused on improving or expanding the feature sets of the core BOSH components. There are several components in the core ecosystem, but in general they will want to understand... - * Scope of features within specific BOSH components. - * API and dependencies between the components of BOSH. - * Teams currently responsible for components, and repositories where ongoing work can be tracked and discussed. +- Scope of features within specific BOSH components. +- API and dependencies between the components of BOSH. +- Teams currently responsible for components, and repositories where ongoing work can be tracked and discussed. Some examples of Internal Developers are... - * BOSH team - comprised of Cloud Foundry Foundation members, the teams work from around the world on different features. - * OSS Community members - those who send pull requests and contributions to the various projects. - +- BOSH team - comprised of Cloud Foundry Foundation members, the teams work from around the world on different features. +- OSS Community members - those who send pull requests and contributions to the various projects. ### CPI Developer A CPI Developer is primarily interested in seeing support for a particular IaaS (like Amazon Web Services and Google Cloud Platform) within BOSH. They will want to understand... - * The API exposed by the director for management of IaaS resources, such as disks and virtual machines. - * The APIs exposed by the IaaS for managing resources. - * Conventions encouraged and enforced by the IaaS around... - * Disaster recovery and failure scenarios. - * Permissions and access control. +- The API exposed by the director for management of IaaS resources, such as disks and virtual machines. +- The APIs exposed by the IaaS for managing resources. +- Conventions encouraged and enforced by the IaaS around... + - Disaster recovery and failure scenarios. + - Permissions and access control. Some examples of CPI Developers are... - * VMware team managing [cloudfoundry/bosh-vsphere-cpi-release](https://github.com/cloudfoundry/bosh-vsphere-cpi-release) - * SAP team managing [cloudfoundry/bosh-openstack-cpi-release](https://github.com/cloudfoundry/bosh-openstack-cpi-release) +- VMware team managing [cloudfoundry/bosh-vsphere-cpi-release](https://github.com/cloudfoundry/bosh-vsphere-cpi-release) +- SAP team managing [cloudfoundry/bosh-openstack-cpi-release](https://github.com/cloudfoundry/bosh-openstack-cpi-release) diff --git a/content/post-deploy.md b/content/post-deploy.md index 88a761e70..55e68040e 100644 --- a/content/post-deploy.md +++ b/content/post-deploy.md @@ -1,3 +1,5 @@ +# Post-deploy + (See [Job Lifecycle](job-lifecycle.md) for an explanation of when post-deploy scripts run.) !!! note @@ -9,6 +11,7 @@ Release job can have a post-deploy script that will run after all jobs in the deployments successfully started (and ran post-start scripts). This script allows the job to execute any additional commands against a whole deployment before considering deploy finished. --- + ## Director Configuration {: #director-configuration } !!! note @@ -17,6 +20,7 @@ Release job can have a post-deploy script that will run after all jobs in the de For Directors older than 280.0.23, the `director.enable_post_deploy` property was configurable and if not set, defaulted to true. Prior to 268.7.0, the property defaulted to false. --- + ## Job Configuration {: #job-configuration } To add a post-deploy script to a release job: @@ -34,6 +38,7 @@ templates: ``` --- + ## Script Implementation {: #script-implementation } Post-deploy script is usually just a regular shell script. Since post-deploy script is executed in a similar way as other release job scripts (start, stop, drain scripts) you can use job's package dependencies. @@ -53,6 +58,7 @@ Post-deploy scripts in a deployment are executed in parallel. Post-deploy scripts run at a lower CPU scheduling priority than the BOSH agent to keep the agent responsive. See [Job Lifecycle](job-lifecycle.md) for details. --- + ## Logs {: #logs } You can find logs for each release job's post-deploy script in the following locations: diff --git a/content/post-start.md b/content/post-start.md index 0a5e466fb..bf8d91f29 100644 --- a/content/post-start.md +++ b/content/post-start.md @@ -1,3 +1,5 @@ +# Post-start + (See [Job Lifecycle](job-lifecycle.md) for an explanation of when post-start scripts run.) !!! note @@ -9,6 +11,7 @@ Release job can have a post-start script that will run after the job is started (specifically after monit successfully starts a process). This script allows the job to execute any additional commands against a machine and/or persistent data before considering release job as successfully started. --- + ## Job Configuration {: #job-configuration } To add a post-start script to a release job: @@ -26,6 +29,7 @@ templates: ``` --- + ## Script Implementation {: #script-implementation } Post-start script is usually just a regular shell script. Since post-start script is executed in a similar way as other release job scripts (start, stop, drain scripts) you can use job's package dependencies. @@ -45,6 +49,7 @@ Post-start scripts in a single deployment job (typically is composed of multiple Post-start scripts run at a lower CPU scheduling priority than the BOSH agent to keep the agent responsive. See [Job Lifecycle](job-lifecycle.md) for details. --- + ## Logs {: #logs } You can find logs for each release job's post-start script in the following locations: diff --git a/content/post-stop.md b/content/post-stop.md index 2591cd7af..17bfb3f3c 100644 --- a/content/post-stop.md +++ b/content/post-stop.md @@ -1,3 +1,5 @@ +# Post-stop + (See [Job Lifecycle](job-lifecycle.md) for an explanation of when post-stop scripts run.) !!! note @@ -6,6 +8,7 @@ Release job can have a post-stop script that will run when the job is restarted or stopped. This script will run following a monit stop for all jobs on the VM in parallel. --- + ## Job Configuration {: #job-configuration } To add a post-stop script to a release job: @@ -23,6 +26,7 @@ templates: ``` --- + ## Script Implementation {: #script-implementation } Post-stop script is usually just a regular shell script. Since post-start script is executed in a similar way as other release job scripts (start, stop, drain scripts) you can use job's package dependencies. diff --git a/content/pre-start.md b/content/pre-start.md index af097789b..9d84fa475 100644 --- a/content/pre-start.md +++ b/content/pre-start.md @@ -1,3 +1,5 @@ +# Pre-start + (See [Job Lifecycle](job-lifecycle.md) for an explanation of when pre-start scripts run.) !!! note @@ -9,6 +11,7 @@ Release job can have a pre-start script that will run before the job is started. This script allows the job to prepare machine and/or persistent data before starting its operation. For example, when writing a release for Cassandra, each node will need to migrate format of SSTables. That procedure may be lengthy and should happen before the node can successfully start. --- + ## Job Configuration {: #job-configuration } To add a pre-start script to a release job: @@ -26,6 +29,7 @@ templates: ``` --- + ## Script Implementation {: #script-implementation } Pre-start script is usually just a regular shell script. ERB tags may be used for templating. Since pre-start script is executed in a similar way as other release job scripts (start, stop, drain scripts) you can use job's package dependencies. @@ -48,6 +52,7 @@ Pre-start scripts in a single deployment job (typically is composed of multiple Pre-start scripts run at a lower CPU scheduling priority than the BOSH agent to keep the agent responsive. See [Job Lifecycle](job-lifecycle.md) for details. --- + ## Logs {: #logs } You can find logs for each release job's pre-start script in the following locations: diff --git a/content/pre-stop.md b/content/pre-stop.md index 9b20ea4fb..72978414b 100644 --- a/content/pre-stop.md +++ b/content/pre-stop.md @@ -1,3 +1,5 @@ +# Pre-stop + (See [Job Lifecycle](job-lifecycle.md) for an explanation of when pre-stop scripts run.) !!! note @@ -8,6 +10,7 @@ However unlike the drain script, during execution of the pre-stop script the rel This new information is exposed to enable the release author to better manage their jobs before an eventual shutdown or a restart. These environment variables are explained further below. --- + ## Job Configuration {: #job-configuration } To add a pre-stop script to a release job: @@ -25,6 +28,7 @@ templates: ``` --- + ## Script Implementation {: #script-implementation } A pre-stop script is usually just a regular shell script. Since the pre-start script is executed in a similar way to other release job scripts (start, stop, drain scripts) you can use the job's package dependencies. @@ -35,23 +39,23 @@ The pre-stop script also uses an exit code to indicate its success (exit code 0) Pre-stop scripts run at a lower CPU scheduling priority than the BOSH agent to keep the agent responsive. See [Job Lifecycle](job-lifecycle.md) for details. --- + ## Environment Variables {: #environment-variables } Pre-stop script can access the following environment variables: -* `BOSH_VM_NEXT_STATE` either `keep` if the VM will be the same VM that start is called on after the stop process, or `delete` if after stop process the VM will be deleted. -* `BOSH_INSTANCE_NEXT_STATE` either `keep` if the instance will be unaffected by the stop process, or `delete` if the instance is deleted after stop process is completed. If this is set to `delete`, the VM will be deleted and a replacement for it will not be created. -* `BOSH_DEPLOYMENT_NEXT_STATE` either `keep` if the deployment will remain after the stop process, or `delete` if the deployment is going to be deleted after completion of the stop process. +- `BOSH_VM_NEXT_STATE` either `keep` if the VM will be the same VM that start is called on after the stop process, or `delete` if after stop process the VM will be deleted. +- `BOSH_INSTANCE_NEXT_STATE` either `keep` if the instance will be unaffected by the stop process, or `delete` if the instance is deleted after stop process is completed. If this is set to `delete`, the VM will be deleted and a replacement for it will not be created. +- `BOSH_DEPLOYMENT_NEXT_STATE` either `keep` if the deployment will remain after the stop process, or `delete` if the deployment is going to be deleted after completion of the stop process. All possible cases of these environment variables: -| Values | Possible Causes | -| - | - | -|BOSH_VM_NEXT_STATE = keep
BOSH_INSTANCE_NEXT_STATE = keep
BOSH_DEPLOYMENT_NEXT_STATE = keep | Something on the VM is being updated
The VM will be kept | -|BOSH_VM_NEXT_STATE = delete
BOSH_INSTANCE_NEXT_STATE = keep
BOSH_DEPLOYMENT_NEXT_STATE = keep | Stemcell update or VM recreate| -|BOSH_VM_NEXT_STATE = delete
BOSH_INSTANCE_NEXT_STATE = delete
BOSH_DEPLOYMENT_NEXT_STATE = keep | Scaling down this instance| -|BOSH_VM_NEXT_STATE = delete
BOSH_INSTANCE_NEXT_STATE = delete
BOSH_DEPLOYMENT_NEXT_STATE = delete | Removing the entire deployment| - +| Values | Possible Causes | +|---------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------| +| `BOSH_VM_NEXT_STATE = keep`
`BOSH_INSTANCE_NEXT_STATE = keep`
`BOSH_DEPLOYMENT_NEXT_STATE = keep` | Something on the VM is being updated
The VM will be kept | +| `BOSH_VM_NEXT_STATE = delete`
`BOSH_INSTANCE_NEXT_STATE = keep`
`BOSH_DEPLOYMENT_NEXT_STATE = keep` | Stemcell update or VM recreate | +| `BOSH_VM_NEXT_STATE = delete`
`BOSH_INSTANCE_NEXT_STATE = delete`
`BOSH_DEPLOYMENT_NEXT_STATE = keep` | Scaling down this instance | +| `BOSH_VM_NEXT_STATE = delete`
`BOSH_INSTANCE_NEXT_STATE = delete`
`BOSH_DEPLOYMENT_NEXT_STATE = delete` | Removing the entire deployment | !!! note If `BOSH_DEPLOYMENT_NEXT_STATE` is set to `delete` then one can safely conclude that consequently both instance and its VM will also be deleted. Similarly, when `BOSH_INSTANCE_NEXT_STATE` is set to `delete` then the corresponding VM also will be deleted after the stop process. diff --git a/content/problems.md b/content/problems.md index 899ca004f..d12e58834 100644 --- a/content/problems.md +++ b/content/problems.md @@ -1,3 +1,5 @@ +# Project Goals + BOSH allows individual developers and teams to easily version, package and deploy software in a reproducible manner. Any software, whether it is a simple static site or a complex multi-component service, will need to be updated and repackaged at some point. This updated software might need to be deployed to a cluster, or it might need to be packaged for end-users to deploy to their own servers. In a lot of cases, the developers who produced the software will be deploying it to their own production environment. Usually, a team will use a staging, development, or demo environment that is similarly configured to their production environment to verify that updates run as expected. These staging environments are often taxing to build and administer. Maintaining consistency between multiple environments is often painful to manage. diff --git a/content/props-common.md b/content/props-common.md index b197468bf..f07d8aecf 100644 --- a/content/props-common.md +++ b/content/props-common.md @@ -1,12 +1,14 @@ +# Recommended Property Types + ## TLS configuration {: #tls } Following is a _suggested_ set of properties for TLS configuration: -* **tls** [Hash]: TLS configuration section. - * **enabled** [Boolean, optional]: Enable/disable TLS. Default should be `true`. - * **cert** [Hash]: Value described by [`certificate` variable type](variable-types.md#certificate). Default is `nil`. - * **protocols** [String, optional]: Space separated list of protocols to support. Example: `TLSv1.2`. - * **ciphers** [String, optional]: OpenSSL formatted list of ciphers to support. Example: `!DES:!RC4:!3DES:!MD5:!PSK`. +- **tls** [Hash]: TLS configuration section. + - **enabled** [Boolean, optional]: Enable/disable TLS. Default should be `true`. + - **cert** [Hash]: Value described by [`certificate` variable type](variable-types.md#certificate). Default is `nil`. + - **protocols** [String, optional]: Space separated list of protocols to support. Example: `TLSv1.2`. + - **ciphers** [String, optional]: OpenSSL formatted list of ciphers to support. Example: `!DES:!RC4:!3DES:!MD5:!PSK`. Example job spec: @@ -94,14 +96,15 @@ variables: ``` --- + ## Environment proxy configuration {: #env-proxy } Following is a _suggested_ set of properties for environment proxy configuration: -* **env** [Hash] - * **http_proxy** [String, optional]: HTTP proxy that software should use. Default: not specified. - * **https_proxy** [String, optional]: HTTPS proxy that software should use. Default: not specified. - * **no_proxy** [String, optional]: List of comma-separated hosts that should skip connecting to the proxy in software. Default: not specified. +- **env** [Hash] + - **http_proxy** [String, optional]: HTTP proxy that software should use. Default: not specified. + - **https_proxy** [String, optional]: HTTPS proxy that software should use. Default: not specified. + - **no_proxy** [String, optional]: List of comma-separated hosts that should skip connecting to the proxy in software. Default: not specified. Example job spec: diff --git a/content/quick-start.md b/content/quick-start.md index 594ba56ec..f535f9c48 100644 --- a/content/quick-start.md +++ b/content/quick-start.md @@ -1,5 +1,6 @@ -The easiest ways to get started with BOSH is by running on your local workstation with [VirtualBox](https://www.virtualbox.org/). If you are interested in bringing up a director in another environment, like [Google Cloud Platform](https://cloud.google.com/), choose your IaaS from the navigation for more detailed instructions. +# Quick Start +The easiest ways to get started with BOSH is by running on your local workstation with [VirtualBox](https://www.virtualbox.org/). If you are interested in bringing up a director in another environment, like [Google Cloud Platform](https://cloud.google.com/), choose your IaaS from the navigation for more detailed instructions. ## Prerequisites @@ -11,7 +12,6 @@ Before trying to deploy the Director, make sure you have satisfied the following 1. Install [VirtualBox](https://www.virtualbox.org/wiki/Downloads). - ## Install First, create a workspace for our `virtualbox` environment. This directory will keep some state and configuration files that we will need. @@ -35,11 +35,11 @@ Now, we can run the [`virtualbox/create-env.sh`](https://github.com/cloudfoundry During the bootstrap process, you will see a few stages: - * Creating BOSH Director - dependencies are downloaded, the VM is created, and BOSH is installed, configured, and started. - * Adding Network Routes - a route to the virtual network is added to ensure you will be able to connect to BOSH-managed VMs. - * Generating `.envrc` - a settings file is generated so you can easily connect to the environment later. - * Configuring Environment Alias - an alias is added for the `bosh` command so you can reference the environment as `vbox`. - * Updating Cloud Config - default settings are applied to the Director so you can easily deploy software later. +- Creating BOSH Director - dependencies are downloaded, the VM is created, and BOSH is installed, configured, and started. +- Adding Network Routes - a route to the virtual network is added to ensure you will be able to connect to BOSH-managed VMs. +- Generating `.envrc` - a settings file is generated so you can easily connect to the environment later. +- Configuring Environment Alias - an alias is added for the `bosh` command so you can reference the environment as `vbox`. +- Updating Cloud Config - default settings are applied to the Director so you can easily deploy software later. After a few moments, BOSH should be started. To verify, first load your connection settings, and then run your first `bosh` command where you should see similar output. @@ -65,7 +65,6 @@ Congratulations - BOSH is running! Now you're ready to [deploy](#deploy) !!! help "Troubleshooting" If you run into any trouble, please continue to the [VirtualBox Troubleshooting](bosh-lite.md) section. - ## Deploy Run through quick steps below or follow [deploy workflow](basic-workflow.md) that goes through the same steps but with more explanation. diff --git a/content/rackhd-cpi.md b/content/rackhd-cpi.md index 4055c628c..0701f17dd 100644 --- a/content/rackhd-cpi.md +++ b/content/rackhd-cpi.md @@ -1,3 +1,5 @@ +# RackHD CPI + [RackHD CPI](https://bosh.io/releases/github.com/cloudfoundry-incubator/bosh-rackhd-cpi-release) works with [OpenStack raw stemcells](https://bosh.io/stemcells/bosh-openstack-kvm-ubuntu-xenial-go_agent-raw). See more details on Github: [https://github.com/cloudfoundry-incubator/bosh-rackhd-cpi-release]. diff --git a/content/recent.md b/content/recent.md index 8bbdaa3f5..6811cd358 100644 --- a/content/recent.md +++ b/content/recent.md @@ -1,46 +1,48 @@ +# Recent Changes + Removed referenced to `bosh-init` --- -* [Configs](configs.md) -* VM Resources on [instance groups](manifest-v2.md#instance-groups) -* [Job Templates](job-templates.md) +- [Configs](configs.md) +- VM Resources on [instance groups](manifest-v2.md#instance-groups) +- [Job Templates](job-templates.md) --- -* [Package vendoring](package-vendoring.md) -* [Native DNS Support](dns.md) +- [Package vendoring](package-vendoring.md) +- [Native DNS Support](dns.md) --- -* [CLI v2](cli-v2.md) -* [CPI config](cpi-config.md) +- [CLI v2](cli-v2.md) +- [CPI config](cpi-config.md) --- -* [Addon placement rules](runtime-config.md#addons) -* [Director-wide tagging](runtime-config.md#tags) -* [Deployment tagging](manifest-v2.md#tags) +- [Addon placement rules](runtime-config.md#addons) +- [Director-wide tagging](runtime-config.md#tags) +- [Deployment tagging](manifest-v2.md#tags) --- -* [Link properties](links-properties.md) -* [Manual linking](links-manual.md) -* [Explicit ARP Flushing](flush-arp.md) -* [Events](events.md) -* [Access event logging](director-access-events.md) -* [Customizing persistent disk FS](persistent-disk-fs.md) +- [Link properties](links-properties.md) +- [Manual linking](links-manual.md) +- [Explicit ARP Flushing](flush-arp.md) +- [Events](events.md) +- [Access event logging](director-access-events.md) +- [Customizing persistent disk FS](persistent-disk-fs.md) --- -* [Common addons](addons-common.md) +- [Common addons](addons-common.md) --- -* [Manifest v2](manifest-v2.md) -* [Cloud config](cloud-config.md) -* [AZs](azs.md) -* [Links](links.md) -* [Runtime config](runtime-config.md) -* [Renaming/migrating jobs](migrated-from.md) -* [Persistent and orphaned disks](persistent-disks.md) +- [Manifest v2](manifest-v2.md) +- [Cloud config](cloud-config.md) +- [AZs](azs.md) +- [Links](links.md) +- [Runtime config](runtime-config.md) +- [Renaming/migrating jobs](migrated-from.md) +- [Persistent and orphaned disks](persistent-disks.md) diff --git a/content/recover.md b/content/recover.md index 8d5274b4d..2c04df72d 100644 --- a/content/recover.md +++ b/content/recover.md @@ -1,3 +1,5 @@ +# Deployment Recovery + !!! note Requires director v277.4.0 and CLI v7.3.0 @@ -16,11 +18,9 @@ similar to [Cloud Check](cck.md), with several exceptions: `cloud-check` command uses the `max_in_flight` values in the deployment manfiest. - Otherwise, the types of [problems](cck.md#problems) and the mechanism by which they are repaired are the same as in the `cloud-check` command. - ## Creating a recovery plan {: #create-recovery-plan } To create a recovery plan, invoke the `bosh create-recovery-plan` command like: @@ -29,7 +29,7 @@ To create a recovery plan, invoke the `bosh create-recovery-plan` command like: bosh create-recovery-plan recovery-plan.yml ``` -```text +```shell Task 223 Task 223 | 17:17:43 | Scanning 9 VMs: Checking VM states (00:00:31) @@ -155,8 +155,8 @@ Each recovery plan has the following schema: **instance_groups_plan** [Array, required]: The name of instance groups in the deployment to recover. -* **max_in_flight_override** [Integer or Percentage, optional]: The `max_in_flight` value to use for problem resolution in the given instance group. -* **planned_resolutions** [Hash, optional]: Specifies which resolution to pick per problem type. Example: `{missing_vm: recreate_vm_without_wait, unresponsive_agent: reboot}` +- **max_in_flight_override** [Integer or Percentage, optional]: The `max_in_flight` value to use for problem resolution in the given instance group. +- **planned_resolutions** [Hash, optional]: Specifies which resolution to pick per problem type. Example: `{missing_vm: recreate_vm_without_wait, unresponsive_agent: reboot}` Here is an example of a complete recovery plan, generated from the above session: @@ -186,7 +186,7 @@ Using the recovery plan above, invoking `bosh recover` looks like: bosh recover recovery-plan.yml ``` -```text +```shell Task 225 Task 225 | 17:35:49 | Scanning 9 VMs: Checking VM states (00:00:31) diff --git a/content/release-blobs.md b/content/release-blobs.md index b18caa60b..061de1213 100644 --- a/content/release-blobs.md +++ b/content/release-blobs.md @@ -1,3 +1,5 @@ +# Working with Blobs + !!! note Examples use CLI v2. @@ -52,6 +54,7 @@ bosh add-blob ~/Downloads/cockroach-latest.linux-amd64.tgz cockroach-latest.linu - updates `config/blobs.yml` to start tracking blobs --- + ## Listing blobs {: #listing-blobs } To list currently tracked blobs use `bosh blobs` command: @@ -73,6 +76,7 @@ Succeeded Blobs that have not been uploaded to release blobstore will be marked as `local` until they are uploaded. --- + ## Uploading blobs {: #saving-blobs } Blobs should be saved into release blobstore before cutting a new final release so that others can rebuild a release at a future time. @@ -85,6 +89,7 @@ Blobs should be saved into release blobstore before cutting a new final release `config/blobs.yml` should be checked into a Git repository. --- + ## Removing blobs {: #removing-blobs } Once a blob is no longer needed by a package it can be stopped being tracked. diff --git a/content/release-blobstore.md b/content/release-blobstore.md index cd5fe0e3d..93bc50c56 100644 --- a/content/release-blobstore.md +++ b/content/release-blobstore.md @@ -1,3 +1,5 @@ +# Release Blobstore + !!! note Examples require CLI v2. @@ -15,7 +17,7 @@ and `local`. S3 provider is used for most production releases. It's can be used with any S3-compatible blobstore (in compatibility mode) like Google Cloud Storage and Swift. -**config/final.yml** +### config/final.yml ```yaml --- @@ -25,7 +27,7 @@ blobstore: bucket_name: ``` -**config/private.yml** +### config/private.yml ```yaml --- @@ -41,7 +43,7 @@ See [Configuring S3 release blobstore](s3-release-blobstore.md) for details and Google Cloud Storage can be used without S3 compatibility mode. -**config/final.yml** +### config/final.yml ```yaml --- @@ -51,7 +53,7 @@ blobstore: bucket_name: ``` -**config/private.yml** +### config/private.yml By default, your [Application Default Credentials](https://cloud.google.com/docs/authentication/production#providing_credentials_to_your_application) will be used. Alternatively, create a `config/private.yml` file to use a separate JSON key. When using a separate JSON key, ensure that the service account has the privilege "Storage Legacy Bucket Owner" for the GCS bucket: @@ -65,11 +67,12 @@ blobstore: ``` --- + ## Local Configuration {: #local-config } Local provider is useful for testing. -**config/final.yml** +### config/final.yml ```yaml --- @@ -87,7 +90,7 @@ Nothing in `config/private.yml`. Azure Storage Account is supported from bosh version `278.0.0`. -**config/final.yml** +### config/final.yml ```yaml --- @@ -98,7 +101,7 @@ blobstore: account_name: ``` -**config/private.yml** +### config/private.yml ```yaml --- @@ -108,6 +111,7 @@ blobstore: ``` --- + ## Release Compression Configuration {: #no-compression } !!! note "Version Requirements" @@ -117,7 +121,7 @@ blobstore: You can control whether the outer release tarball is compressed by setting the `no_compression` flag in `config/final.yml`. -**config/final.yml** +### config/final.yml ```yaml --- @@ -129,7 +133,7 @@ blobstore: no_compression: true ``` -* **no_compression** [Boolean, optional]: When set to `true`, disables compression for the outer release tarball. Defaults to `false` (compression enabled) if not specified. +- **no_compression** [Boolean, optional]: When set to `true`, disables compression for the outer release tarball. Defaults to `false` (compression enabled) if not specified. !!! note The `bosh export-release` command does not currently respect the `no_compression` flag due to technical limitations. When using `bosh export-release`, the outer tarball will always be compressed regardless of the `no_compression` setting in `final.yml`. diff --git a/content/release-urls.md b/content/release-urls.md index 2d151dd3f..75259879c 100644 --- a/content/release-urls.md +++ b/content/release-urls.md @@ -1,3 +1,5 @@ +# Release URLs + This topic describes allowed types of URLs for downloading releases (typically found in deployment manifests). @@ -21,6 +23,7 @@ Above declaration is equivalent to running `bosh upload-release syslog-11.tgz --sha1 332ac15609b220a3fdf5efad0e0aa069d8235788`. --- + ## HTTP/HTTPs URLs {: #http } Upload Release tarball from remote location. @@ -40,6 +43,7 @@ Above declaration is equivalent to running `bosh upload-release https://bosh.io/d/github.com/cloudfoundry/syslog-release?v=11 --sha1 332ac15609b220a3fdf5efad0e0aa069d8235788`. --- + ## Git over HTTP/HTTPs URLs {: #git-http } Reconstruct Release tarball from remote Git repository. diff --git a/content/release.md b/content/release.md index 89385a07b..bd0221692 100644 --- a/content/release.md +++ b/content/release.md @@ -1,3 +1,5 @@ +# Release Management - Overview + A release is a versioned collection of configuration properties, configuration templates, start up scripts, source code, binary artifacts, and anything else required to build and deploy software in a reproducible way. A release is the layer placed on top of a [stemcell](stemcell.md). They are self-contained and provide very specific software for the purpose of that release. For example, a Redis release might include start-up and shutdown scripts for `redis-server`, a tarball with Redis source code obtained from the Redis official website, and a few configuration properties allowing cluster operators to alter that Redis configuration. diff --git a/content/remove-dev-tools.md b/content/remove-dev-tools.md index bbbdcbdc4..5926684ab 100644 --- a/content/remove-dev-tools.md +++ b/content/remove-dev-tools.md @@ -1,3 +1,5 @@ +# Removing Dev Tools from VMs + !!! note This feature is available with bosh-release v255.4+ and on 3213+ stemcell series. diff --git a/content/repack-stemcell.md b/content/repack-stemcell.md index 834a97d6a..1e328df77 100644 --- a/content/repack-stemcell.md +++ b/content/repack-stemcell.md @@ -1,3 +1,5 @@ +# Repacking Stemcells + !!! note Applies to CLI v2.0.12+. @@ -11,6 +13,7 @@ The [CLI v2](cli-v2.md) includes a command to repack stemcells; this enables lim - cloud properties --- + ## Syntax {: #syntax } ```shell @@ -42,8 +45,8 @@ The `repack-stemcell` command can be used to enable the encryption of the root f Two arguments enable the encryption of the root filesystem: -* **encrypted** [Boolean, optional]: Must be set to `true` if encryption of the root filesystem -* **kms\_key\_arn** [String, optional]: Created in the [Encryption Keys](https://console.aws.amazon.com/iam/home#encryptionKeys) section of the Identity and Access Management (IAM) console. If not specified _and_ `encrypted` is true, the root filesystem will be encrypted with the default key. +- **encrypted** [Boolean, optional]: Must be set to `true` if encryption of the root filesystem +- **kms\_key\_arn** [String, optional]: Created in the [Encryption Keys](https://console.aws.amazon.com/iam/home#encryptionKeys) section of the Identity and Access Management (IAM) console. If not specified _and_ `encrypted` is true, the root filesystem will be encrypted with the default key. We modify the cloud-properties of an AWS stemcell to encrypt the root filesystem of instances deployed with our repacked stemcell. The cloud-properties must be specified as valid JSON. This only works with heavy stemcells: @@ -70,7 +73,7 @@ curl -L https://bosh.io/d/stemcells/bosh-google-kvm-ubuntu-xenial-go_agent | tar Should result in: -```text +```shell % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 100 137 100 137 0 0 268 0 --:--:-- --:--:-- --:--:-- 268 diff --git a/content/resurrector.md b/content/resurrector.md index d01286b76..27388ff4f 100644 --- a/content/resurrector.md +++ b/content/resurrector.md @@ -1,3 +1,5 @@ +# Auto-healing Capabilities - Resurrector + The Resurrector is a plugin to the [Health Monitor](bosh-components.md#health-monitor). It's responsible for automatically recreating VMs that become inaccessible. The Resurrector continuously cross-references VMs expected to be running against the VMs that are sending heartbeats. When resurrector does not receive heartbeats for a VM for a certain period of time, it will kick off a task on the Director (scan and fix task) to try to "resurrect" that VM. The Director may do one of two things: @@ -13,6 +15,7 @@ Resurrection can be turned off per specific deployment job instance or for all V The Health Monitor deploys with the Resurrector plugin disabled by default. To use it, you must enable the Resurrector plugin in your BOSH deployment manifest. --- + ## Enabling and Configuring the Resurrector {: #enable } To enable the Resurrector: @@ -26,9 +29,9 @@ To enable the Resurrector: ``` 1. Optionally change configuration values: - * **minimum\_down\_jobs** [Integer, optional]: If the total number of instances that are down in a deployment (within time interval T) is below this number, the Resurrector will _always_ request to fix instances. This decision takes precedence to the `percent_threshold` check when the # of down instances ≤ `minimum_down_jobs`. Default is 5. - * **percent_threshold** [Float, optional]: If the percentage of instances that are down in a deployment (within time interval T) is greater than the threshold percentage, the Resurrector will _not_ request to fix any instance. Going over this threshold is called "meltdown". Default is 0.2 (20%). - * **time_threshold** [Integer, optional]: Time interval (in seconds) used in the above calculations. Default is 600. + - **minimum\_down\_jobs** [Integer, optional]: If the total number of instances that are down in a deployment (within time interval T) is below this number, the Resurrector will _always_ request to fix instances. This decision takes precedence to the `percent_threshold` check when the # of down instances ≤ `minimum_down_jobs`. Default is 5. + - **percent_threshold** [Float, optional]: If the percentage of instances that are down in a deployment (within time interval T) is greater than the threshold percentage, the Resurrector will _not_ request to fix any instance. Going over this threshold is called "meltdown". Default is 0.2 (20%). + - **time_threshold** [Integer, optional]: Time interval (in seconds) used in the above calculations. Default is 600. ```yaml properties: @@ -42,60 +45,59 @@ To enable the Resurrector: 1. Depending on how you configured [Director user management](director-users.md), credentials are specified in the `user` and `password` properties or using a custom `client` to authenticate with the UAA. - ### Option a) Using UAA User Management {: #uaa-client } +### Option a) Using UAA User Management {: #uaa-client } - Define an additional client in the `uaa.clients` section of your manifest: +Define an additional client in the `uaa.clients` section of your manifest: - ```yaml - properties: - uaa: - clients: - hm: - override: true - authorized-grant-types: client_credentials - scope: "" - authorities: bosh.admin - secret: "hm-password" - ``` +```yaml +properties: + uaa: + clients: + hm: + override: true + authorized-grant-types: client_credentials + scope: "" + authorities: bosh.admin + secret: "hm-password" +``` - Configure the Health Monitor to use the client with the defined secret to authenticate with the Director: +Configure the Health Monitor to use the client with the defined secret to authenticate with the Director: - ```yaml - properties: - hm: - director_account: - client_id: hm - client_secret: "hm-password" - ``` +```yaml +properties: + hm: + director_account: + client_id: hm + client_secret: "hm-password" +``` - ### Option b) Using Preconfigured Users {: #preconfigured-users } +### Option b) Using Preconfigured Users {: #preconfigured-users } - Create new Director user so that the Resurrector plugin can communicate with the Director and query/submit information about deployments. +Create new Director user so that the Resurrector plugin can communicate with the Director and query/submit information about deployments. - ```yaml - properties: - director: - user_management: - provider: local - local: - users: - - {name: admin, password: admin-password} - - {name: hm, password: hm-password} - ``` +```yaml +properties: + director: + user_management: + provider: local + local: + users: + - {name: admin, password: admin-password} + - {name: hm, password: hm-password} +``` - Configure the Health Monitor to use the HM user and password to authenticate with the Director: +Configure the Health Monitor to use the HM user and password to authenticate with the Director: - ```yaml - properties: - hm: - director_account: - user: hm - password: hm-password - ``` +```yaml +properties: + hm: + director_account: + user: hm + password: hm-password +``` 1. Deploy. - ### Customizing for Your Deployment {: #customize } For most deployments, you can use the default configuration values. In very small or very large deployments, the default values may need customization, as discussed in the following examples. @@ -109,6 +111,7 @@ If your deployment consists of only five VMs, you may not want the Resurrector t If your deployment consists of 1000 VMs, and you use the defaults, the Resurrector notifies the Director to recreate at least five VMs and up to 200 VMs. Depending on your deployment, you may consider even 100 down instances a catastrophic failure. In this scenario, set `percent_threshold` to 5% so that the Director resurrects 50 instances or fewer. --- + ## Enabling the Resurrector with Resurrection Config {: #enable-with-resurrection-config } !!! tip "Beta Feature" @@ -123,7 +126,6 @@ The resurrection state will be updated directly after updating the resurrection 3. Upload a resurrection config, which enables resurrection. 4. Missing VM will be resurrected. - ### Structure of resurrection config ```yaml @@ -149,18 +151,16 @@ rules: - `rules` - a list of resurrection rules. When resurrection rules are interpreted for a given instance, all resurrection rules from all resurrection configs are considered for matching. If no rule matches, resurrection config considers resurrection as `enabled`. - `enabled` - a boolean which enables (`true`) or disables (`false`) resurrection. -- `exclude` *(Optional)* - a resurrection rule which will result in the resurrection configuration being overridden wherever the specified constraints do not match. -- `include` *(Optional)* - a resurrection rule which will result in the resurrection configuration being overridden wherever the specified constraints match. -- `deployments` *(Optional)* - a list of deployments which can be used as a filter for the include/exclude directives. -- `instance_groups` *(Optional)* - a list of instance groups which can be used as a filter for the include/exclude directives. - +- `exclude` _(Optional)_ - a resurrection rule which will result in the resurrection configuration being overridden wherever the specified constraints do not match. +- `include` _(Optional)_ - a resurrection rule which will result in the resurrection configuration being overridden wherever the specified constraints match. +- `deployments` _(Optional)_ - a list of deployments which can be used as a filter for the include/exclude directives. +- `instance_groups` _(Optional)_ - a list of instance groups which can be used as a filter for the include/exclude directives. ### General -* When specifying both properties, `instance_groups` and `deployments`, the rule is only applied for the instance groups of the specified deployments. -* When only `instance_groups` is specified, the rule will be applied to all matching instance groups across **all** deployments. -* Multiple rules are evaluated by the `&` operator. This means, if one rule with `enabled: false` matches for a given instance resurrection for this instance is disabled. - +- When specifying both properties, `instance_groups` and `deployments`, the rule is only applied for the instance groups of the specified deployments. +- When only `instance_groups` is specified, the rule will be applied to all matching instance groups across **all** deployments. +- Multiple rules are evaluated by the `&` operator. This means, if one rule with `enabled: false` matches for a given instance resurrection for this instance is disabled. ### Examples @@ -168,40 +168,41 @@ By default, resurrection is turned on and the following examples demonstrate how 1. Disable resurrection for a deployment `dep1` by creating a `resurrection.yml`: - ```yaml - rules: - - enabled: false - include: - deployments: - - dep1 - ``` - Running `bosh update-config --type resurrection --name disable-dep1 resurrection.yml` disables resurrection for all deployments in the include block, i.e. `dep1`. For all other deployments, resurrection is still enabled. + ```yaml + rules: + - enabled: false + include: + deployments: + - dep1 + ``` + + Running `bosh update-config --type resurrection --name disable-dep1 resurrection.yml` disables resurrection for all deployments in the include block, i.e. `dep1`. For all other deployments, resurrection is still enabled. 2. Disable resurrection for an instance group `instance-group-1` of a deployment `dep1` by creating a `resurrection.yml`: - ```yaml - rules: - - enabled: false - include: - deployments: - - dep1 - instance_groups: - - instance-group-1 - ``` + ```yaml + rules: + - enabled: false + include: + deployments: + - dep1 + instance_groups: + - instance-group-1 + ``` - Running `bosh update-config --type resurrection --name disable-dep1-instance-group-1 resurrection.yml` disables resurrection for all instance groups of all the specified deployments, i.e. `instance-group-1` of deployment `dep1`. For all other instance groups and deployments, resurrection is still enabled. + Running `bosh update-config --type resurrection --name disable-dep1-instance-group-1 resurrection.yml` disables resurrection for all instance groups of all the specified deployments, i.e. `instance-group-1` of deployment `dep1`. For all other instance groups and deployments, resurrection is still enabled. 3. Disable resurrection for all deployments except for deployment `dep1` by creating a `resurrection.yml`: - ```yaml - rules: - - enabled: false - exclude: - deployments: - - dep1 - ``` - Running `bosh update-config --type resurrection --name disable-all-but-dep1 resurrection.yml` disables resurrection for all deployments except deployment `dep-1`. + ```yaml + rules: + - enabled: false + exclude: + deployments: + - dep1 + ``` + Running `bosh update-config --type resurrection --name disable-all-but-dep1 resurrection.yml` disables resurrection for all deployments except deployment `dep-1`. ### Disabling resurrection globally @@ -212,6 +213,7 @@ bosh update-resurrection off ``` Resurrection can then be re-enabled on all deployments with: + ```sh bosh update-resurrection on ``` @@ -226,6 +228,7 @@ rules: Both the CLI command and resurrection config options are meant to temporarily disable resurrection while you diagnose other issues. You should update the deployment manifest to permanently [disable the Resurrector](#disable). --- + ## Disabling the Resurrector {: #disable } To disable the Resurrector: @@ -243,6 +246,7 @@ To disable the Resurrector: 1. Optionally remove Director user created for the Health Monitor. --- + ## Viewing the Resurrector's Activity {: #audit } The Resurrector creates scan and fix tasks on the Director using the Health Monitor user. Since these are normal tasks, you can use the `tasks` CLI command to view them. @@ -258,4 +262,3 @@ Similarly, to view finished Resurrector activity, run the following and look for ```sh bosh tasks --recent --all -d '' ``` - diff --git a/content/runtime-config.md b/content/runtime-config.md index f5abac465..3633b7b87 100644 --- a/content/runtime-config.md +++ b/content/runtime-config.md @@ -1,9 +1,12 @@ +# Using Runtime Config - Overview + !!! note This feature is available with bosh-release v255.4+. The Director has a way to specify global configuration for all VMs in all deployments. The runtime config is a YAML file that defines IaaS agnostic configuration that applies to all deployments. --- + ## Updating and retrieving runtime config {: #update } To update runtime config on the Director use [`bosh update-runtime-config`](sysadmin-commands.md#cloud-config) CLI command. @@ -37,6 +40,7 @@ tags: Once runtime config is updated all deployments will be considered outdated. `bosh deployments` does not currently show that but we have plans to show that information. The Director will apply runtime config changes to each deployment during the next `bosh deploy` for that deployment. --- + ## Example {: #example } ```yaml @@ -67,14 +71,15 @@ addons: [] ``` --- + ## Releases Block {: #releases } **releases** [Array, required]: Specifies the releases used by the addons. -* **name** [String, required]: Name of a release name used by an addon. -* **version** [String, required]: The version of the release to use. Version *cannot* be `latest`; it must be specified explicitly. -* **url** [String, optional]: URL of a release to download. Works with CLI v2. Example: `https://bosh.io/d/github.com/cloudfoundry/syslog-release?v=11`. -* **sha1** [String, optional]: SHA1 of asset referenced via URL. Works with CLI v2. Example: `332ac15609b220a3fdf5efad0e0aa069d8235788`. +- **name** [String, required]: Name of a release name used by an addon. +- **version** [String, required]: The version of the release to use. Version *cannot* be `latest`; it must be specified explicitly. +- **url** [String, optional]: URL of a release to download. Works with CLI v2. Example: `https://bosh.io/d/github.com/cloudfoundry/syslog-release?v=11`. +- **sha1** [String, optional]: SHA1 of asset referenced via URL. Works with CLI v2. Example: `332ac15609b220a3fdf5efad0e0aa069d8235788`. See [Release URLs](release-urls.md) for more details. @@ -97,6 +102,7 @@ releases: ``` --- + ## Addons Block {: #addons } Operators typically want to ensure that certain software runs on all VMs managed by the Director. Examples of such software are: @@ -110,28 +116,28 @@ An addon is a release job that is colocated on all VMs managed by the Director. **addons** [Array, optional]: Specifies the [addons](terminology.md#addon) to be applied to all deployments. -* **name** [String, required]: A unique name used to identify and reference the addon. -* **jobs** [Array of hashes, requires]: Specifies the name and release of release jobs to be colocated. - * **name** [String, required]: The job name. - * **release** [String, required]: The release where the job exists. - * **properties** [Hash, optional]: Specifies job properties. Properties allow the Director to configure jobs to a specific environment. -* **include** [Hash, optional]: Specifies inclusion [placement rules](#placement-rules) Available in bosh-release v260+. -* **exclude** [Hash, optional]: Specifies exclusion [placement rules](#placement-rules). Available in bosh-release v260+. +- **name** [String, required]: A unique name used to identify and reference the addon. +- **jobs** [Array of hashes, requires]: Specifies the name and release of release jobs to be colocated. + - **name** [String, required]: The job name. + - **release** [String, required]: The release where the job exists. + - **properties** [Hash, optional]: Specifies job properties. Properties allow the Director to configure jobs to a specific environment. +- **include** [Hash, optional]: Specifies inclusion [placement rules](#placement-rules) Available in bosh-release v260+. +- **exclude** [Hash, optional]: Specifies exclusion [placement rules](#placement-rules). Available in bosh-release v260+. ### Placement Rules for `include` and `exclude` Directives {: #placement-rules } Available rules: -* **stemcell** [Array of hashes, optional]: at least one of the items must match - * **os** [String, required]: Matches stemcell's operating system. Example: `ubuntu-xenial` -* **deployments** [Array of strings, optional]: Matches based on deployment names. -* **jobs** [Array of hashes, optional]: at least one of the configured jobs must match - * **name** [String, required]: Matching job name. - * **release** [String, required]: Matching release name. -* **instance_groups** [Array of strings, optional]: Matches based on instance group names. Available in bosh-release v268+. -* **lifecycle** [String, optional]: Matches based on lifecycle type, either `service` or `errand`. Note that a service VM is any non-errand VM and that colocated errands may actually run on service VMs. Available in bosh-release v262+. -* **networks** [Array of strings, optional]: Matches based on network names. Available in bosh-release v262+. -* **teams** [Array of strings, optional]: Matches based on team names. Available in bosh-release v253+. +- **stemcell** [Array of hashes, optional]: at least one of the items must match + - **os** [String, required]: Matches stemcell's operating system. Example: `ubuntu-xenial` +- **deployments** [Array of strings, optional]: Matches based on deployment names. +- **jobs** [Array of hashes, optional]: at least one of the configured jobs must match + - **name** [String, required]: Matching job name. + - **release** [String, required]: Matching release name. +- **instance_groups** [Array of strings, optional]: Matches based on instance group names. Available in bosh-release v268+. +- **lifecycle** [String, optional]: Matches based on lifecycle type, either `service` or `errand`. Note that a service VM is any non-errand VM and that colocated errands may actually run on service VMs. Available in bosh-release v262+. +- **networks** [Array of strings, optional]: Matches based on network names. Available in bosh-release v262+. +- **teams** [Array of strings, optional]: Matches based on team names. Available in bosh-release v253+. Example: @@ -176,7 +182,6 @@ The rules inside a directive are also based on the AND operator; if the VM meets the criteria of all of the rules, it will be considered `included` or `excluded`. All items within the arrays in an inclusion/exclusion rule use the OR operator. - For example, given the following config: ```yaml @@ -206,7 +211,7 @@ except for any VMs with `job1` belonging to deployment `dep1` or `dep2`. In pseudocode: -``` +```sql AND ( INCLUDE ( AND ( @@ -230,6 +235,7 @@ In pseudocode: ``` --- + ## Tags Block {: #tags } **tags** [Hash, optional]: Specifies key value pairs to be sent to the CPI for VM tagging. Combined with deployment level tags during the deploy. Available in bosh-release v260+. diff --git a/content/s3-release-blobstore.md b/content/s3-release-blobstore.md index 533e0f291..5614e050d 100644 --- a/content/s3-release-blobstore.md +++ b/content/s3-release-blobstore.md @@ -1,3 +1,5 @@ +# Using S3 & IAM Policies + !!! note Examples require CLI v2. @@ -52,7 +54,7 @@ blobstore: ``` !!! note - The .gitignore file in the BOSH release should include config/private.yml. This file should not be committed to the release repo. It is only meant for the release maintainers. config/final.yml, on the other hand, should not be in the .gitignore file, and should be committed to the repository, as it is for users consuming and deploying the release. + The `.gitignore` file in the BOSH release should include `config/private.yml`. This file should **not** be committed to the release repo. It is only meant for the release maintainers. `config/final.yml`, on the other hand, should not be in the `.gitignore` file, and should be committed to the repository, as it is for users consuming and deploying the release. - Attach a _user_ policy that would limit the user to permissions to read/write to the bucket that was just created: diff --git a/content/sample-manifest.md b/content/sample-manifest.md index 2752008b5..97714eb9c 100644 --- a/content/sample-manifest.md +++ b/content/sample-manifest.md @@ -1,3 +1,5 @@ +# Sample Manifest + The following is a sample BOSH deployment manifest. See [Understanding the BOSH Deployment Manifest](deployment-manifest.md) for an explanation of the manifest contents. ```yaml diff --git a/content/scheduled-procs.md b/content/scheduled-procs.md index 183497ab6..883c54a1f 100644 --- a/content/scheduled-procs.md +++ b/content/scheduled-procs.md @@ -1,3 +1,5 @@ +# Scheduled Processes + Jobs can use cron installed on the stemcell to schedule processes. In a [pre-start script](pre-start.md), copy a script from your job's `bin` directory to one of the following locations: - `/etc/cron.hourly` diff --git a/content/scheduled-task-cleanup.md b/content/scheduled-task-cleanup.md index dd5f726d7..6696ce457 100644 --- a/content/scheduled-task-cleanup.md +++ b/content/scheduled-task-cleanup.md @@ -1,23 +1,33 @@ -# Scheduled task-cleanup +# Scheduled Task Cleanup + +## Scheduled task-cleanup + Scheduled task-cleanup is a scheduled task in bosh director, to get rid of the completed tasks from the database and disk and to avoid filling the disk with the log files. This task is scheduled to run every 7 days by default and keep last 2000 tasks for each type. ## Configuration + ### `director.tasks_cleanup_schedule` + The schedule for the task-cleanup job. The default value is `0 0 0 */7 * * UTC` which means the task-cleanup job will run once every 7 days at midnight UTC. ### `director.max_tasks` + The maximum number of tasks per each type to keep in disk. The default value is `2000`. ### `director.tasks_retention_period` + The generic retention period for tasks and their log files. The default value is `''` and the scheduled job will only consider `max_tasks`. The tasks will be cleaned up if one of the `max_tasks` or `tasks_retention_period` is reached. The configuration is an integer with unit day. ### `director.tasks_deployments_retention_period` + The retention period for tasks and their log files of specific deployments. The default value is `''` and the scheduled job will only consider `max_tasks`. The tasks will be cleaned up if one of the `max_tasks`, `tasks_retention_period` or `tasks_deployments_retention_period` is reached. The configuration is an array of map, with retention unit day and deployment name. -### Example configuration: + +### Example configuration + ```yaml properties: director: diff --git a/content/snapshots.md b/content/snapshots.md index 77bdbafd2..26878873b 100644 --- a/content/snapshots.md +++ b/content/snapshots.md @@ -1,3 +1,5 @@ +# Taking Snapshots + !!! note This feature is experimental. @@ -38,19 +40,19 @@ Once you enable snapshots in your deployment, you can use following CLI commands bosh snapshots ``` -Displays the job, Content ID (CID), and created date of all snapshots. Run bosh snapshots to display a list of CIDs if you need to find specific snapshots to recover. +Displays the job, Content ID (CID), and created date of all snapshots. Run `bosh snapshots` to display a list of CIDs if you need to find specific snapshots to recover. ```shell bosh take snapshot [JOB] [INDEX] ``` -Takes a snapshot of the job VM that you specify. If you do not specify a JOB, takes a snapshot of every VM in the current deployment. +Takes a snapshot of the job VM that you specify. If you do not specify a `JOB`, takes a snapshot of every VM in the current deployment. ```shell bosh delete snapshot SNAPSHOT-CID ``` -Deletes the snapshot that SNAPSHOT-CID specifies. +Deletes the snapshot that `SNAPSHOT-CID` specifies. ```shell bosh delete snapshots @@ -75,7 +77,7 @@ To schedule snapshots for all VMs in all deployments: 1. Add a [cron-formatted](https://github.com/jmettraux/rufus-scheduler/blob/two/README.rdoc#a-note-about-cron-jobs) schedule as a value for the `snapshot_schedule` key. - ```yaml + ``yaml properties: director: enable_snapshots: true diff --git a/content/stemcell.md b/content/stemcell.md index 980a1989e..34ac08e97 100644 --- a/content/stemcell.md +++ b/content/stemcell.md @@ -1,3 +1,5 @@ +# What is a Stemcell? + A stemcell is a versioned Operating System image wrapped with IaaS specific packaging. @@ -47,8 +49,8 @@ their respective versions. We have a versioning system that resembles semver. Using an example stemcell version, `621.45`: -* `621` is the major version number. -* `45` is the patch version number. +- `621` is the major version number. +- `45` is the patch version number. The minor version is absent. @@ -61,11 +63,11 @@ version. The schedule for stemcells roughly looks like: -* New LTS distributions from Canonical are consumed around every 2-3 years. This +- New LTS distributions from Canonical are consumed around every 2-3 years. This is usually an overhaul on how the bosh-agent interacts with the base operating system. -* New patches are cut every 3 weeks to pick up any low & medium CVEs published - by https://usn.ubuntu.com. A patch will also be cut within a week of a high or +- New patches are cut every 3 weeks to pick up any low & medium CVEs published + by [Ubuntu Security Notices](https://usn.ubuntu.com). A patch will also be cut within a week of a high or critical CVE being fixed. **What are the differences between stemcell lines?** @@ -78,15 +80,14 @@ stemcell, which has compatibility considerations with the BOSH director. **How is a stemcell is built and how one would go about building their own stemcell?** -The code lives on GitHub at -https://github.com/cloudfoundry/bosh-linux-stemcell-builder. Building a stemcell +The code lives on GitHub at [bosh-linux-stemcell-builder](https://github.com/cloudfoundry/bosh-linux-stemcell-builder). Building a stemcell occurs in stages. Each stage is represented as a BASH script and can be found in `stemcell_builder/stages//apply.sh`. Each IaaS has its own list of stages defined here: -https://github.com/cloudfoundry/bosh-linux-stemcell-builder/blob/master/bosh-stemcell/lib/bosh/stemcell/stage_collection.rb. +[stage_collection.rb](https://github.com/cloudfoundry/bosh-linux-stemcell-builder/blob/master/bosh-stemcell/lib/bosh/stemcell/stage_collection.rb). ### Links -* [CI Source Repo](https://github.com/cloudfoundry/bosh-stemcells-ci) -* [Stemcell Builder](https://github.com/cloudfoundry/bosh-linux-stemcell-builder) -* [Stemcell Hardening](https://techdocs.broadcom.com/us/en/vmware-tanzu/platform/tanzu-operations-manager/3-0/tanzu-ops-manager/security-pcf-infrastructure-stemcell-index.html) +- [CI Source Repo](https://github.com/cloudfoundry/bosh-stemcells-ci) +- [Stemcell Builder](https://github.com/cloudfoundry/bosh-linux-stemcell-builder) +- [Stemcell Hardening](https://techdocs.broadcom.com/us/en/vmware-tanzu/platform/tanzu-operations-manager/3-0/tanzu-ops-manager/security-pcf-infrastructure-stemcell-index.html) diff --git a/content/sysadmin-commands.md b/content/sysadmin-commands.md index ff3ebdd63..273aa5ddd 100644 --- a/content/sysadmin-commands.md +++ b/content/sysadmin-commands.md @@ -1,9 +1,12 @@ +# CLI v1 Commands + !!! note See [CLI v2](cli-v2.md) for an updated CLI. This document lists the all CLI commands you use to perform system administration tasks. --- + ## For Cluster Operators {: #cluster-operators } Use these commands against the Director to manage deployments and associated assets. @@ -29,6 +32,7 @@ bosh status [--uuid] Displays the configuration file and deployment manifest in use, and information about the BOSH Director such as name, URL, version, current username, UUID, and CPI. --- + ### Users {: #user } Use these commands to create and delete users on the Director. @@ -46,6 +50,7 @@ bosh delete user [USERNAME] Deletes a specific user from the BOSH Director. Prompts you for a `USERNAME` if you omit this information. --- + ### Releases {: #dir-release } ```shell @@ -80,6 +85,7 @@ bosh export release NAME/VERSION OS/VERSION Exports given release as a tarball, including compiled packages for stemcell that matches OS and version. --- + ### Stemcells {: #dir-stemcells } ```shell @@ -102,6 +108,7 @@ bosh delete stemcell NAME VERSION [--force] Deletes a stemcell and all associated compiled packages. Fails if any deployment references this stemcell. --- + ### Cloud config {: #cloud-config } ```shell @@ -117,6 +124,7 @@ bosh update cloud-config FILE_PATH Updates currently saved cloud config in the Director. See [cloud config description](cloud-config.md). --- + ### Runtime config {: #runtime-config } ```shell @@ -132,6 +140,7 @@ bosh update runtime-config FILE_PATH Updates currently saved runtime config in the Director. See [runtime config description](runtime-config.md). --- + ### Deployment {: #deployment } ```shell @@ -165,6 +174,7 @@ bosh delete deployment DEPLOYMENT_NAME [--force] Deletes job instances, VMs, disks, snapshots, templates associated with the deployment `DEPLOYMENT_NAME`. --- + ### Job/VM Health {: #health } ```shell @@ -221,6 +231,7 @@ bosh cck [DEPLOYMENT_NAME] [--auto] [--report] Scans for differences between the VM state database that the Director maintains and the actual state of the VMs. For each difference the scan detects, `bosh cck` offers possible repair options. --- + ### Errands {: #errand } ```shell @@ -236,6 +247,7 @@ bosh run errand ERRAND_NAME [--download-logs] [--logs-dir DESTINATION_DIRECTORY] Instructs the BOSH Director to run the named errand on a job instance on a VM. --- + ### SSH {: #ssh } ```shell @@ -251,6 +263,7 @@ bosh ssh JOB [INDEX] [COMMANDS] When you provide arguments without an option flag, the Director executes the arguments as commands on the job VM. For example, `bosh ssh redis 0 "ls -R"` runs the `ls -R` command on the redis/0 job VM. --- + ### Director Tasks {: #tasks } ```shell @@ -272,6 +285,7 @@ bosh task [TASK_ID] [--event] [--cpi] [--debug] [--result] [--raw] Displays the status of a task that you specify and tracks its output. You can track only one of the following log types at a time: event, CPI, debug, or result. Defaults to event. --- + ### Logs {: #logs } ```shell @@ -281,6 +295,7 @@ bosh logs JOB [INDEX] [--agent] [--job] [--only filter1,filter2,...] [--dir DEST Fetches a job or agent log from a VM. Supports custom filtering only for job logs. --- + ### Events {: #events } ```shell @@ -290,6 +305,7 @@ bosh events [--before-id ID] [--deployment NAME] [--task ID] [--instance NAME/ID Displays table that lists events based on filters specified. See [events details](events.md). --- + ### Disks {: #disks } ```shell @@ -305,6 +321,7 @@ bosh attach disk DISK_ID INSTANCE_NAME/ID Attaches persistent disk to given instance. Instance must be in the stopped state. --- + ### Snapshots {: #snapshots } ```shell @@ -332,6 +349,7 @@ bosh delete snapshot SNAPSHOT_CID Deletes a snapshot. --- + ## For Release Maintainers {: #release-maintainers } Use these commands against the Director to create and update releases. diff --git a/content/terminology.md b/content/terminology.md index 05bfdc015..d1bb6e852 100644 --- a/content/terminology.md +++ b/content/terminology.md @@ -1,118 +1,143 @@ +# Terminology + ## AZ or Availability Zone {: #az } An availability zone represents a separated set of cloud resources (typically compute, networking and storage) such that failures in one AZ cause minimal impact in a different AZ. [See usage details](azs.md). --- + ## Addon {: #addon } A release job that is colocated on all VMs managed by the Director. Addons are configured in the runtime config. [See usage details](runtime-config.md). --- + ## Agent {: #agent } A process that runs continuously on each VM that BOSH deploys (one Agent process per VM). The Agent executes tasks in response to messages it receives from the Director. --- + ## bosh-init {: #bosh-init } Tool that was replaced by CLI v2's `create-env` command used for creating and updating. --- + ## BOSH Lite {: #bosh-lite } BOSH Lite (aka bosh-lite) is a Director VM that is configured to use Warden CPI, which emulates VMs with containers. It's typically installed locally with VirtualBox; however, it could also be installed onto any cloud BOSH supports. [See usage details](bosh-lite.md) --- + ## Canary (Instance) {: #canary } Canary instances are first instances updated within an instance group. Any update error in a canary instance causes the deployment to stop. Since only canaries are affected before an update stops, problem jobs and packages are prevented from taking over all instances. --- + ## CLI (v1) {: #cli } The BOSH Command Line Interface (CLI) is what you use to run BOSH commands. You must [install](bosh-cli.md) the CLI to use BOSH. Run `bosh help --all` to view the help. It is superseded by CLI v2. --- + ## CLI v2 {: #cli-v2 } The BOSH Command Line Interface (CLI) is what you use to run BOSH commands. CLI v2 is a new major version of CLI. It also replaces bosh-init CLI to manage Director VM. It's the recommended way to interact with the Director. [See usage details](cli-v2.md). --- + ## Cloud {: #cloud } Same as Infrastructure as a Service. --- + ## Cloud ID (CID) {: #cid } ID returned from the Cloud identifying particular resource such as VM or disk. --- + ## Cloud Config {: #cloud-config } The cloud config is a YAML file that defines IaaS specific configuration used by the Director and all deployments. It allows to separate IaaS specific configuration into its own file and keep deployment manifests IaaS agnostic. [See usage details](cloud-config.md). --- + ## Compiled Release {: #compiled-release } A compiled release contains jobs and compiled packages. A non-compiled release (or just release) contains jobs and source packages. [See usage details](compiled-releases.md). --- + ## CPI {: #cpi } A Cloud Provider Interface is an abstraction layer between the Director and an IaaS (cloud). CPIs have to implement a small number of methods to perform VM, disk and network operations. CPIs could be written in different languages. --- + ## Deploy {: #deploy } BOSH deploys software to the cloud using a deployment manifest, one or more stemcells, and one or more releases. --- + ## Deployment {: #deployment } An encapsulation of software and configuration that BOSH can deploy to the cloud. You can think of a deployment as the state of a collection of VMs: what software is on them, what resources they use, and how these are orchestrated. Even though BOSH creates the deployment using ephemeral resources, the deployment is stable in that BOSH re-creates VMs that fail and otherwise works to keep your software running. BOSH also manages persistent disks so that state (for example, database data files) can survive when VMs are re-created. Combination of a deployment manifest, stemcells, and releases is portable across different clouds with minimal changes to the deployment manifest. See [What is a Deployment?](deployment.md). --- + ## Deployment manifest (or just manifest) {: #manifest } A YAML file that identifies one or more releases, stemcells and specifies how to configure them for a given deployment. --- + ## Director {: #director } The main BOSH component that coordinates the Agents and responds to user requests and system events. The Director is the orchestrator of deployments. --- + ## Director Blobstore {: #director-blobstore } A repository where BOSH stores release artifacts, logs, stemcells, and other content, at various times during the lifecycle of a BOSH release. --- + ## Director Task {: #director-task } The basic unit of work performed by the Director. You can get the status and logs for any task. You can monitor the task throughout its lifecycle, which progresses through states like queued, processing, done, and error. --- + ## Director VM {: #director-vm } A single VM with the Director and other necessary components. --- + ## Disk Pool {: #disk-pool } See [Disk Type](#disk-type). --- + ## Disk Type {: #disk-type } Disk type is a named disk configuration specified in the cloud config. [See usage details](cloud-config.md#disk-types) and [read more about persistent disks](persistent-disks.md). --- + ## Environment {: #environment } An environment consists of a Director and deployments that it orchestrates. A good example of two separate environments are staging and production environments. --- + ## Errand {: #errand } An errand is a short-lived job that can be triggered by an operator any time after the deploy. Examples: @@ -124,6 +149,7 @@ An errand is a short-lived job that can be triggered by an operator any time aft [See details](errands.md). --- + ## Event {: #event } Actions taken by the Director (via user or system control) are recorded as events to the Director database. Examples: @@ -134,121 +160,145 @@ Actions taken by the Director (via user or system control) are recorded as event [See details](events.md). --- + ## IaaS {: #iaas } Short for Infrastructure as a Service. BOSH enables the Cloud Foundry PaaS and other software deployed with BOSH to support multiple IaaS providers. --- + ## Ignored Instances {: #ignored-instances } Ignored Instances are not updated during a deploy. However they are not fully excluded from the deploy process as their state might be needed for providing [links](#links). If an Ignored Instance does not have a VM during a deploy, the VM will be recreated which can lead to template rendering problems due to the way variable sets are chosen. --- + ## Instance {: #instance } An instance corresponds to a single VM that performs specific jobs. Each instance is a part of an instance group. --- + ## Instance Group (previously known as Deployment Job) {: #instance-group } An instance group is a collection of instances tasked to perform same jobs. Each instance group has an associated VM type, persistent disk type, a stemcell and a set of jobs. Instance groups are configured in the deployment manifest. --- + ## Instance Lifecycle {: #instance-lifecycle } Stages that all jobs (and their associated processes) go through during a deployment process on one instance. For example: pre-start, start, drain, etc. [See details](job-lifecycle.md). --- + ## Job (aka Release Job) {: #job } A job is part of a release. It contains startup, shutdown scripts, and configuration files that tell the Agent how to start, run and monitor software on a VM. Jobs can depend on packages for necessary software. [See details](jobs.md). --- + ## Job Lifecycle {: #job-lifecycle } Stages that all jobs (and their associated processes) go through during a deployment process on one instance. For example: pre-start, start, drain, etc. [See details](job-lifecycle.md). --- + ## Jumpbox {: #jumpbox } A VM that acts as a single access point for the Director and deployed VMs. For resilience, there should be more than one jump box. Allowing access through jump boxes and disabling direct access to the other VMs is a common security measure. --- + ## Links {: #links } Links provide a mechanism for [Jobs](#job) to share deploy time instance data with each other. [See details](links.md). --- + ## MicroBOSH {: #microbosh } See [Director VM](#director-vm). --- + ## Operator {: #operator } A user that sets up and/or uses the Director (via BOSH CLI or Director API) to manage cloud resources. --- + ## Operations file (ops file) {: #operations-file } A YAML file that includes multiple operations to be applied to a different YAML file. Several CLI commands such as `create-env` and `interpolate` allow to provide multiple operations files via `--ops-file` flag. [See details](cli-ops-files.md). --- + ## Operation {: #operation } A single directive in an operations file. An operation describes one change to make to a YAML structure. Currently there are two types of operations: replace and remove. [See details](cli-ops-files.md). --- + ## Orphaned (Persistent) Disk {: #orphaned-disk } An orphaned disk is a persistent disk that will be garbage collected after a few days unless it's reattached to an instance. --- + ## Package {: #package } A package is part of a release. It contains vendored in software source and scripts to compile it. Packages can depend on other packages. --- + ## Persistent Disk {: #persistent-disk } A persistent disk is a disk created in the cloud and associated with a specific [instance](terminology.md#instance). While instance's associated VM is recreated, same persistent disk will be reattached. [See usage details](persistent-disks.md). --- + ## Release {: #release } A collection of configuration files, source code, jobs, packages and accompanying information needed to make a software component deployable by BOSH. A self-contained release should have no dependencies that need to be fetched from the internet. See [What is a Release?](release.md). --- + ## Resource Pool {: #resource-pool } Resource pool is collections of VMs created from the same stemcell, with the same configuration, in a deployment. --- + ## Runtime Config {: #runtime-config } The runtime config is a YAML file that defines global configuration used by the Director and all deployments. It allows to specify addons. [See usage details](runtime-config.md). --- + ## Stemcell {: #stemcell } A generic VM image that BOSH clones and configures during deployment. A stemcell is a template from which BOSH creates whatever VMs are needed for a wide variety of components and products. See [What is a Stemcell?](stemcell.md). --- + ## Team {: #team } Each deployment can be managed by specific teams. A logged in UAA user can belong to one or more teams. [See details](director-bosh-teams). --- + ## Variable (var) {: #variable } Variable points to a saved value in some store. Variables are typically used in configuration files (manifests) to decouple sensitive (passwords, certificates) or volatile (bucket name, number of instances) data from more static content (general configuration). Variables are denoted with double parens -- `((namespace/var-name))`. --- + ## VM Extension {: #vm-extension } VM extension is a named Virtual Machine configuration in the cloud config that allows to specify arbitrary IaaS specific configuration such as associated security groups and load balancers. [See usage details](cloud-config.md#vm-extensions). --- + ## VM Type {: #vm-type } VM type is a named Virtual Machine size configuration in the cloud config. [See usage details](cloud-config.md#vm-types). diff --git a/content/tips.md b/content/tips.md index d9017c9ac..b15eb6cdc 100644 --- a/content/tips.md +++ b/content/tips.md @@ -1,3 +1,5 @@ +# Troubleshooting + This document lists several common problems. If you are looking for CPI specific errors see: - [AWS CPI errors](aws-cpi-errors.md) @@ -6,6 +8,7 @@ This document lists several common problems. If you are looking for CPI specific - [vSphere CPI errors](vsphere-cpi-errors.md) --- + ## Getting logs from an unresponsive VM {: #unresponsive-vm-logs } If the BOSH agent on a VM becomes unresponsive, the `bosh logs` command will not work. This can make it difficult to diagnose what is causing the agent to become unresponsive. @@ -34,8 +37,7 @@ For instructions on how to launch a new VM, shutdown or reboot VMs, attach or de bosh deploy ``` -```text - +```shell Failed creating bound missing vms > cloud_controller_worker/0: Timed out pinging to 013ce5c9-e7fc-4f1d-ac24 after 600 seconds (00:16:03) Failed creating bound missing vms > uaa/0: Timed out pinging to b029652d-14c3-4d68-98c7 after 600 seconds (00:16:12) Failed creating bound missing vms > uaa/0: Timed out pinging to 1f56ddd1-7f2d-4afc-ae43 after 600 seconds (00:16:23) @@ -53,6 +55,7 @@ For a more detailed error report, run: bosh task 45 --debug This problem can occur due to: - blocked network connectivity between the Agent on a new VM and NATS (typically the Director VM). For stemcells released after April 2022 additional restrictions can block connectivity to NATS. Specifically, traffic to the NATS endpoint is restricted to membership in a specific **cgroup**, this membership is based on the process id. To add your SSH session's pid to the cgroup, follow these steps: + ```shell # To add your SSH Session to the NATS Access CGROUP you can execute the below steps. # This will work EXCLUSIVELY for the SSH Session that executes the command and does @@ -68,6 +71,7 @@ This problem can occur due to: # Connection to 10.0.0.6 4222 port [tcp/*] succeeded! # INFO {....} ``` + - bootstrapping problem on the VM and/or wrong configuration of the Agent - blocked or slow boot times of the VM @@ -89,14 +93,14 @@ In case when the VM spec has not been applied, the instance name is not availabl The steps for remediation are similar to the [Timed out pinging to ... after 600 seconds](#unreachable-agent) case, where operators should try to SSH into VMs as the problem is occurring so they can look at the Agent logs. --- + ## ...is not running after update {: #failed-job } ```shell bosh deploy ``` -```text - +```shell Started updating job access_z1 > access_z1/0 (canary) Done updating job route_emitter_z1 > route_emitter_z1/0 (canary) (00:00:13) Done updating job cc_bridge_z1 > cc_bridge_z1/0 (canary) (00:00:20) @@ -115,6 +119,7 @@ This problem occurs when one of the release jobs on a VM did not successfully st This problem may also arise when deployment manifest specifies too small of a [canary/update watch time](deployment-manifest.md#update) which may not be large enough for a process to successfully start. --- + ## umount: /var/vcap/store: device is busy {: #unmount-persistent-disk } ```text @@ -127,13 +132,14 @@ L Error: Action Failed get_task: Task 5be893c6-7a2c-4f3f-420b-433fd23528a1 resul This process occurs when one of the processes (from one of the installed jobs) did not fully shutdown and continues to retain a reference to the persistent disk. Agent tries to unmount persistent disk and unmount command fails. You can use `bosh ssh` command to get inside the VM and diagnose which process is holding onto the persistent disk via `lsof | grep /var/vcap/store` (ran as root). --- + ## Running command: bosh-blobstore-dav -c ... 500 Internal Server Error {: #blobstore-out-of-space } ```shell bosh deploy ``` -```text +```shell ... Failed compiling packages > dea_next/3e95ef8425be45468e044c05cc9aa65494281ab5: Action Failed get_task: Task bd35f7c1-2144-4045-763e-40beeafc9fa3 result: Compiling package dea_next: Uploading compiled package: Creating blob in inner blobstore: Making put command: Shelling out to bosh-blobstore-dav cli: Running command: 'bosh-blobstore-dav -c /var/vcap/bosh/etc/blobstore-dav.json put /var/vcap/data/tmp/bosh-platform-disk-TarballCompressor-CompressFilesInDir949066221 cd91a1c5-a034-4c69-4608-6b18cc3fcb2b', stdout: 'Error running app - Putting dav blob cd91a1c5-a034-4c69-4608-6b18cc3fcb2b: Wrong response code: 500; body: @@ -151,6 +157,7 @@ This problem can occur if the Director is configured to use built-in blobstore a If `bosh clean-up` command fails with 500 Internal Server Error, consider removing `/var/vcap/store/director/tasks` to free up a little bit of persistent disk space before running `bosh clean-up` command again. (Deleting that directory will delete Director task debug logs.) --- + ## Debugging Director database {: #director-db } It may be necessary to dive into the Director DB. The easiest way to do so is to SSH into the Director VM and use `console`. For example: @@ -187,17 +194,18 @@ echo 'Bosh::Director::Models::Deployment.all' | /var/vcap/jobs/director/bin/cons Bosh uses the [Sequel database toolkit](https://sequel.jeremyevans.net/) to interact with its database. See associated [cheat sheet](https://sequel.jeremyevans.net/rdoc/files/doc/cheat_sheet_rdoc.html) and some sample statements below: Inspecting a deployment by name (equivalent of `select * from deployment where name = "test"`) + ```ruby Bosh::Director::Models::Deployment.where(name: "test").all ``` - Listing persistent disks by Iaas id + ```ruby Bosh::Director::Models::PersistentDisk.map(:disk_cid) ``` -```text +```shell => ["disk-39346983-594c-4682-8183-f3280bc634f5", "disk-8c0cee29-be48-4982-8c07-84c4d30e30a2", "disk-9980e0bb-ce9b-4af6-97bd-d262835bfbf8", ...] @@ -216,13 +224,14 @@ Bosh::Director::Models::PersistentDisk.where(Sequel.~(cpi: "")).map(:disk_cid) ``` --- + ## Task X cancelled {: #canceled-task } ```shell bosh deploy ``` -```text +```shell ... Started preparing package compilation > Finding packages to compile. Done (00:00:01) @@ -236,13 +245,14 @@ Task 106 cancelled This problem typically occurs if the Director's system time is out of sync, or if the Director machine is underpowered. --- + ## Upload release fails {: #upload-release-entity-too-large } ```shell bosh upload release blah.tgz ``` -```text +```shell ... Started creating new packages > blah_package/f9098f452f46fb072a6000b772166f349ffe27da. Failed: Could not create object, 413/ 413 Request Entity Too Large @@ -266,13 +276,14 @@ Error 100: Could not create object, 413/ This failure occurs due to nginx configuration problems with the director and the nginx blobstore. This can be remedied by configuring the `max_upload_size` property on the director and blobstore jobs. --- -## Persistent Disk with id not found {: #persistent-disk-not-found } + +## Persistent Disk with id `` not found {: #persistent-disk-not-found } ```shell bosh create-env ``` -```text +```shell (...) Command 'deploy' failed: Deploying: @@ -288,6 +299,7 @@ Command 'deploy' failed: The SSH tunnel between your machine and the VM in the cloud can be terminated prematurely, see [corresponding bug](https://github.com/cloudfoundry/bosh-cli/issues/110). Update CLI v2 to version >= v2.0.2 to fix this. --- + ## Errors creating or fetching credhub variables {: #variables-permission} Upgrading to Credhub 2.0.0 introduces a breaking change with permissions. Users no longer have automatic implicit read/write access for new paths. Any new credentials the director may generate as part of a deployment will not be created, and any variables the user did not previously have access to will be inaccessible. diff --git a/content/troubleshooting.md b/content/troubleshooting.md index 58560aea5..bdea0ad32 100644 --- a/content/troubleshooting.md +++ b/content/troubleshooting.md @@ -1,9 +1,12 @@ +# Failing VMs + This document describes the usual actions and tools used to drill down failing VMs issues, and find a root cause. For troubleshooting specific issues, see also [those tips](tips.md). --- + ## Troubleshooting a failed deployment {: #failed-vms } These are usual steps to do in order to drill down to the root cause for some @@ -19,7 +22,7 @@ VM instance failure. the `$PATH`. 4. Check failing Monit jobs with `monit summary`. Whenever the failure has - happened at [`pre-atart` stage](job-lifecycle.md#start), this list is empty + happened at [`pre-start` stage](job-lifecycle.md#start), this list is empty because Monit configuration is not yet assembled. 5. Check for any full disk device with `df -h`. diff --git a/content/trusted-certs.md b/content/trusted-certs.md index 4e0a00888..7db12a911 100644 --- a/content/trusted-certs.md +++ b/content/trusted-certs.md @@ -1,9 +1,12 @@ +# Installing Certificates on VMs + !!! note This feature is available with bosh-release v176+ (1.2992.0) and stemcells v2992+. This document describes how to configure the Director to add a set of trusted certificates to all VMs managed by that Director. Configured trusted certificates are added to the default certificate store on each VM and will be automatically seen by the majority of software (e.g. curl). --- + ## Configuring Trusted Certificates {: #configure } To configure the Director with trusted certificates: diff --git a/content/understanding-bosh.md b/content/understanding-bosh.md index cdd3ba5fa..5b1db97f0 100644 --- a/content/understanding-bosh.md +++ b/content/understanding-bosh.md @@ -1,3 +1,5 @@ +# Understanding BOSH + BOSH is an open source tool chain for release engineering, deployment, and lifecycle management of large-scale distributed services. @@ -25,9 +27,9 @@ scripts, with a version number that identifies these components. A BOSH release consists of the software packages to be installed and the processes, or jobs, to run on the VMs in a deployment. -* A package contains source code and a script for compiling and installing the +- A package contains source code and a script for compiling and installing the package, with optional dependencies on other packages. -* A job is a set of configuration files and scripts to run the binaries from a +- A job is a set of configuration files and scripts to run the binaries from a package. ### Manifest {: #manifest } @@ -64,15 +66,15 @@ rest. For example: -* To switch a deployment between clouds: - * Keep the same release - * Use a stemcell specific to the new cloud - * Tweak the manifest -* To scale up an application: - * Keep the same release - * Use the same stemcell - * Change one line in the manifest -* To update or roll back an application: - * Use a newer or older release version - * Use the same stemcell - * Use the same manifest +- To switch a deployment between clouds: + - Keep the same release + - Use a stemcell specific to the new cloud + - Tweak the manifest +- To scale up an application: + - Keep the same release + - Use the same stemcell + - Change one line in the manifest +- To update or roll back an application: + - Use a newer or older release version + - Use the same stemcell + - Use the same manifest diff --git a/content/update-cloud-config.md b/content/update-cloud-config.md index 2c56a819d..6e33c82bc 100644 --- a/content/update-cloud-config.md +++ b/content/update-cloud-config.md @@ -1,3 +1,5 @@ +# Updating Cloud Config + !!! note Document uses CLI v2. diff --git a/content/uploading-releases.md b/content/uploading-releases.md index 64b052c1a..4dc0eceff 100644 --- a/content/uploading-releases.md +++ b/content/uploading-releases.md @@ -1,3 +1,5 @@ +# Uploading Releases + !!! note Document uses CLI v2. @@ -24,11 +26,12 @@ Here are a few popular releases: [cf-release](https://bosh.io/releases/github.com/cloudfoundry/cf-release) was popular when it used to provide all software components for Cloud Foundry. Then - [cf-deployment]](https://github.com/cloudfoundry/cf-deployment) emerged + [cf-deployment](https://github.com/cloudfoundry/cf-deployment) emerged with modularized sources of software components, now provided by 30+ Bosh - releases.) + releases. --- + ## Uploading to the Director {: #upload } CLI provides [`bosh upload-release` command](cli-v2.md#upload-release). @@ -74,7 +77,7 @@ Once the command succeeds, you can view all uploaded releases in the Director: bosh -e vbox releases ``` -```text +```shell Using environment '192.168.56.6' as client 'admin' Name Version Commit Hash @@ -93,6 +96,7 @@ See [Release URLs](release-urls.md) for more details on the URLs accepted by `bosh upload-release`. --- + ## Deployment Manifest Usage {: #using } To use an uploaded release in your deployment, update the `releases` section diff --git a/content/uploading-stemcells.md b/content/uploading-stemcells.md index a0df98abf..a014bd06d 100644 --- a/content/uploading-stemcells.md +++ b/content/uploading-stemcells.md @@ -1,3 +1,5 @@ +# Uploading Stemcells + !!! note Document uses CLI v2. @@ -10,6 +12,7 @@ As described earlier, each deployment can reference one or more stemcells. For a The [stemcells section of bosh.io](http://bosh.io/stemcells) lists official stemcells. --- + ## Uploading to the Director {: #upload } CLI provides [`bosh upload-stemcell` command](cli-v2.md#upload-stemcell). @@ -35,7 +38,7 @@ bosh -e vbox stemcells Should result in: -```text +```shell Using environment '192.168.56.6' as client 'admin' Name Version OS CPI CID @@ -49,6 +52,7 @@ Succeeded ``` --- + ## Deployment Manifest Usage {: #using } To use uploaded stemcell in your deployment, add stemcells: diff --git a/content/variable-types.md b/content/variable-types.md index 716b473ad..30c5be055 100644 --- a/content/variable-types.md +++ b/content/variable-types.md @@ -1,3 +1,5 @@ +# CredHub Variable Types + (See [Variable Interpolation](cli-int.md) for introduction.) Currently CLI supports `password`, `certificate`, `rsa`, and `ssh` types whose @@ -19,43 +21,41 @@ Note that `` indicates value obtained via `((var))` variable syntax. [credhub_cred_types]: https://docs.cloudfoundry.org/credhub/credential-types.html#cred-types --- + ## Password {: #password } -**** [String]: Password value. A random string containing a fixed set -of characters: lowercase letters (from `a` to `z`) and figures (from `0` to -`9`). +- **** [String]: Password value. A random string containing a fixed set of characters: lowercase letters (from `a` to `z`) and figures (from `0` to `9`). Generation options: -* **length** [Number, optional]: The length of password to generate. Defaults +- **length** [Number, optional]: The length of password to generate. Defaults to `20` with the Bosh CLI (whereas the default length with CredHub [is `30`][credhub_gen_pwd_opts]). [credhub_gen_pwd_opts]: https://docs.cloudfoundry.org/api/credhub/version/2.9/#_generate_a_password_credential_request_fields --- + ## Certificate {: #certificate } -**** [Hash]: Certificate. +- **** [Hash]: Certificate. -* **ca** [String]: Certificate's CA (PEM encoded). -* **certificate** [String]: Certificate (PEM encoded). -* **private_key** [String]: Private key (PEM encoded). - Since [Sept 8th, 2017][boshcli_priv_key_len], the Bosh CLI generates private - keys which are `3072` bits long, and doesn't provide any parameter for this, - whereas CredHub default [is 2048][credhub_gen_cert_opts]. +- **ca** [String]: Certificate's CA (PEM encoded). +- **certificate** [String]: Certificate (PEM encoded). +- **private_key** [String]: Private key (PEM encoded). + Since [Sept 8th, 2017][boshcli_priv_key_len], the Bosh CLI generates private keys which are `3072` bits long, and doesn't provide any parameter for this, whereas CredHub default [is 2048][credhub_gen_cert_opts]. [boshcli_priv_key_len]: https://github.com/cloudfoundry/config-server/blob/0ef502116cccef2370f333d37abe9748df125e95/types/certificate_generator.go#L60 [credhub_gen_cert_opts]: https://docs.cloudfoundry.org/api/credhub/version/2.9/#_generate_a_certificate_credential_request_fields Generation options: -* **common_name** [String, required]: the Common Name (CN) used in the certificate subject. Example: `foo.com`. -* **organization** [String, optional]: The organization name (O) used in the certificate subject. Defaults to `Cloud Foundry`. -* **alternative_names** [Array, optional]: Subject alternative names. Example: `["foo.com", "*.foo.com"]`. -* **is_ca** [Boolean, required]: Indicates whether this is a CA certificate (root or intermediate). Defaults to `false`. -* **ca** [String, optional]: Specifies name of a CA certificate to use for making this certificate. Can be specified in conjunction with `is_ca` to produce an intermediate certificate. -* **extended\_key\_usage** [Array, optional]: List of extended key usage. Possible values: `client_auth` and/or `server_auth`. Default: `[]` (empty list). Example: `["client_auth"]`. -* **duration** [Number, optional]: Duration in days of generated credential value. Default: `365`. If a minimum duration is configured in CredHub and is greater than the user provided duration, the certificate will be generated using the minimum duration instead. +- **common_name** [String, required]: the Common Name (CN) used in the certificate subject. Example: `foo.com`. +- **organization** [String, optional]: The organization name (O) used in the certificate subject. Defaults to `Cloud Foundry`. +- **alternative_names** [Array, optional]: Subject alternative names. Example: `["foo.com", "*.foo.com"]`. +- **is_ca** [Boolean, required]: Indicates whether this is a CA certificate (root or intermediate). Defaults to `false`. +- **ca** [String, optional]: Specifies name of a CA certificate to use for making this certificate. Can be specified in conjunction with `is_ca` to produce an intermediate certificate. +- **extended\_key\_usage** [Array, optional]: List of extended key usage. Possible values: `client_auth` and/or `server_auth`. Default: `[]` (empty list). Example: `["client_auth"]`. +- **duration** [Number, optional]: Duration in days of generated credential value. Default: `365`. If a minimum duration is configured in CredHub and is greater than the user provided duration, the certificate will be generated using the minimum duration instead. Example: @@ -108,20 +108,22 @@ variables: ``` --- + ## RSA {: #rsa } **** [Hash]: RSA key. The Bosh CLI generates a private key which is `2048` bits long, and doesn't provide any parameter for this. -* **private_key** [String]: Private key (PEM encoded). -* **public_key** [String]: Public key (PEM encoded). +- **private_key** [String]: Private key (PEM encoded). +- **public_key** [String]: Public key (PEM encoded). --- + ## SSH {: #ssh } -**** [Hash]: SSH key. The Bosh CLI generates a RSA private key which is +- **** [Hash]: SSH key. The Bosh CLI generates a RSA private key which is `2048` bits long, and doesn't provide any parameter for this. -* **private_key** [String]: Private key (PEM encoded). -* **public_key** [String]: Public key (OpenSSH format, "ssh-rsa ..."). -* **public\_key\_fingerprint** [String]: Public key's MD5 fingerprint. Example: `c3:ae:51:ec:cb:a8:09:ac:43:fd:84:dd:11:dd:fe:c7`. +- **private_key** [String]: Private key (PEM encoded). +- **public_key** [String]: Public key (OpenSSH format, "ssh-rsa ..."). +- **public_key_fingerprint** [String]: Public key's MD5 fingerprint. Example: `c3:ae:51:ec:cb:a8:09:ac:43:fd:84:dd:11:dd:fe:c7`. diff --git a/content/virtualbox-cpi.md b/content/virtualbox-cpi.md index a82eb58bc..4b57b2d9c 100644 --- a/content/virtualbox-cpi.md +++ b/content/virtualbox-cpi.md @@ -1,3 +1,5 @@ +# VirtualBox CPI Cloud Properties + This topic describes cloud properties for different resources created by the [VirtualBox CPI](https://bosh.io/releases/github.com/cloudfoundry/bosh-virtualbox-cpi-release). VirtualBox CPI works with [vSphere ESXI stemcells](https://bosh.io/stemcells/bosh-vsphere-esxi-ubuntu-xenial-go_agent). ## AZs {: #azs } @@ -12,12 +14,13 @@ azs: ``` --- + ## Networks {: #networks } Schema for `cloud_properties` section used by network subnet: -* **name** [String, required]: Name of the network. Example: `vboxnet0`. Default: `vboxnet0`. -* **type** [String, optional]: Type of the network. See [`VBoxManage modifyvm` networking settings](https://www.virtualbox.org/manual/ch08.html#idp46691722135120) for valid values. Example: `hostonly`. Default: `hostonly`. +- **name** [String, required]: Name of the network. Example: `vboxnet0`. Default: `vboxnet0`. +- **type** [String, optional]: Type of the network. See [`VBoxManage modifyvm` networking settings](https://www.virtualbox.org/manual/ch08.html#idp46691722135120) for valid values. Example: `hostonly`. Default: `hostonly`. Example of manual network: @@ -34,14 +37,15 @@ networks: ``` --- + ## VM Types / VM Extensions {: #vm-types } Schema for `cloud_properties` section: -* **cpus** [Integer, optional]: Number of CPUs. Example: `1`. Default: `1`. -* **memory** [Integer, optional]: RAM in megabytes. Example: `1024`. Default: `512`. -* **ephemeral_disk** [Integer, optional]: Ephemeral disk size in megabytes. Example: `10240`. Default: `5000`. -* **paravirtprovider** [String, optional]: Paravirtual provider type. See [`VBoxManage modifyvm` general settings](https://www.virtualbox.org/manual/ch08.html#idp46691713664256) for valid values. Default: `minimal`. +- **cpus** [Integer, optional]: Number of CPUs. Example: `1`. Default: `1`. +- **memory** [Integer, optional]: RAM in megabytes. Example: `1024`. Default: `512`. +- **ephemeral_disk** [Integer, optional]: Ephemeral disk size in megabytes. Example: `10240`. Default: `5000`. +- **paravirtprovider** [String, optional]: Paravirtual provider type. See [`VBoxManage modifyvm` general settings](https://www.virtualbox.org/manual/ch08.html#idp46691713664256) for valid values. Default: `minimal`. Example of a VM type: @@ -56,6 +60,7 @@ vm_types: ``` --- + ## Disk Types {: #disk-types } Currently the CPI does not support any cloud properties for disks. @@ -69,6 +74,7 @@ disk_types: ``` --- + ## Global Configuration {: #global } The CPI uses individual VirtualBox VMs and disks. Since the CPI can only talk to a single VirtualBox server it can only manage resources on a single machine. @@ -100,6 +106,7 @@ properties: See [virtualbox_cpi job](https://bosh.io/jobs/virtualbox_cpi?source=github.com/cloudfoundry/bosh-virtualbox-cpi-release) for more details. --- + ## Example Cloud Config {: #cloud-config } ```yaml diff --git a/content/vm-anti-affinity.md b/content/vm-anti-affinity.md index 3048a6a11..3103c1597 100644 --- a/content/vm-anti-affinity.md +++ b/content/vm-anti-affinity.md @@ -1,3 +1,5 @@ +# VM Anti-Affinity + For certain deployment jobs, you might want to distribute the instances across multiple physical resources of the IaaS. Even though an IaaS abstracts away the underlying hardware resources, most have specific APIs to configure VM affinity and anti-affinity rules. One popular example of a deployment job that needs this type of configuration is Hadoop Datanode. If multiple Datanode instances are placed on the same physical machine, replicated data becomes unavailable if that machine fails. To make replication useful in this scenario, BOSH allows you to configure the resource pool for a deployment job. You configure VM anti-affinity rules for an IaaS using the `cloud_properties` sub-block of the `resource_pools` block in your [deployment manifest](deployment-manifest.md). @@ -5,16 +7,17 @@ One popular example of a deployment job that needs this type of configuration is Currently only vSphere and OpenStack CPIs provide a way to do so. --- + ## vSphere Configuration {: #vsphere } The vSphere [VM-VM Affinity Rules](https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.resmgmt.doc/GUID-94FCC204-115A-4918-9533-BFC588338ECB.html) feature allows you to specify whether VMs should run on the same host or be kept on separate hosts. As of BOSH version 101 (stemcell 2693), you can configure the vSphere CPI to include all VMs of a specified BOSH resource pool within a single DRS rule and separate the VMs among multiple hosts. The following resource pool and instance group configuration manifest example instructs BOSH to: -* Create two MySQL VMs in the `cl` vSphere cluster. -* Create a `keep-mysql-on-different-hosts` DRS rule in the `cl` vSphere cluster. -* Configure the DRS rule with a `type` that separates the associated VMs onto different hosts (`separate_vms`). -* Associate the VMs with the DRS rule. +- Create two MySQL VMs in the `cl` vSphere cluster. +- Create a `keep-mysql-on-different-hosts` DRS rule in the `cl` vSphere cluster. +- Configure the DRS rule with a `type` that separates the associated VMs onto different hosts (`separate_vms`). +- Associate the VMs with the DRS rule. ```yaml # Assuming that a MySQL release is used... @@ -68,6 +71,7 @@ vm_extensions: ``` --- + ## OpenStack Configuration {: #openstack } OpenStack's [Filter scheduler](http://docs.openstack.org/developer/nova/devref/filter_scheduler.html) allows to customize compute node selection algorithm which determines placement of new VMs. To enforce anti-affinity among VMs, `ServerGroupAntiAffinityFilter` is available: @@ -76,9 +80,9 @@ OpenStack's [Filter scheduler](http://docs.openstack.org/developer/nova/devref/f The following resource pool and job configuration manifest example instructs BOSH to: -* Assume that the server group was created and its UUID is `af09abf2-2283-47d6-f2bd-2932a9ae949c` -* Assume that the server group specifies 'anti-affinity' policy -* Create seven hadoop-datanode VMs and add them to the server group `af09abf2-2283-47d6-f2bd-2932a9ae949c` +- Assume that the server group was created and its UUID is `af09abf2-2283-47d6-f2bd-2932a9ae949c` +- Assume that the server group specifies 'anti-affinity' policy +- Create seven hadoop-datanode VMs and add them to the server group `af09abf2-2283-47d6-f2bd-2932a9ae949c` ```yaml # Assuming that a Hadoop release is used... diff --git a/content/vm-config.md b/content/vm-config.md index 7383a5051..de6d0e737 100644 --- a/content/vm-config.md +++ b/content/vm-config.md @@ -1,13 +1,16 @@ +# VMs - Filesystem Locations + This topic describes important file system locations, configurations and other settings that are true for all the VMs managed by Bosh, also called the “Bosh instances”. BOSH tries to encourage release authors to follow the conventions listed below, so if you find inconsistencies or improper usage please report such problems to appropriate release authors. --- + ## Global Configuration {: #global } -* `/tmp/`: Global temporary directory is limited to 128MB. A well behaved release job that needs scratch space usually sets up its own temporary directory inside ephemeral data directory (e.g. `/var/vcap/data/redis-server/tmp`). +- `/tmp/`: Global temporary directory is limited to 128MB. A well behaved release job that needs scratch space usually sets up its own temporary directory inside ephemeral data directory (e.g. `/var/vcap/data/redis-server/tmp`). -* `vcap` user: Pre-configured user that comes with the stemcells. Release jobs may run processes under that user. The default password will be random. +- `vcap` user: Pre-configured user that comes with the stemcells. Release jobs may run processes under that user. The default password will be random. !!! Note BPM enforces that convention and runs processes with the `vcap` user by @@ -16,7 +19,7 @@ BOSH tries to encourage release authors to follow the conventions listed below, [process Schema](https://bosh.io/docs/bpm/config/#process-schema) section for more details. -* `/etc/logrotate.d/vcap`: Logrotate configuration for `/var/vcap/sys/log/` sub-directories. See the [“Log rotation” section](job-logs.md#log-rotation) for more details on rotation of log files. +- `/etc/logrotate.d/vcap`: Logrotate configuration for `/var/vcap/sys/log/` sub-directories. See the [“Log rotation” section](job-logs.md#log-rotation) for more details on rotation of log files. The contract with release authors, for the log files produced by jobs, is the following: @@ -34,61 +37,62 @@ BOSH tries to encourage release authors to follow the conventions listed below, so, but it's not recommended. --- -## Release Job and Package Directories {: #jobs-and-packages } - -* `/var/vcap/`: VCAP[1] directory contains majority of the configuration settings and associated assets for the Bosh instance. - -* `/var/vcap/packages/`: Contains enabled release packages for the instance. The Agent is responsible for managing which packages are enabled or disabled on the VM. -* `/var/vcap/jobs/`: Contains evaluated release jobs for the instance. The Agent is responsible for managing which jobs are enabled or disabled on the VM. +## Release Job and Package Directories {: #jobs-and-packages } - - `/var/vcap/jobs//bin/`: Conventional location where Bosh renders the templates for wrapper scripts and [hook scripts](job-lifecycle.md). +- `/var/vcap/`: VCAP[1] directory contains majority of the configuration settings and associated assets for the Bosh instance. - - `/var/vcap/jobs//config/`: Conventional location for the job configuration files, like BPM config (in `/var/vcap/jobs/redis-server/config/bpm.yml`, responsible for starting executables). and other rendered config files (e.g. `/var/vcap/jobs/redis-server/config/redis.conf`). +- `/var/vcap/packages/`: Contains enabled release packages for the instance. The Agent is responsible for managing which packages are enabled or disabled on the VM. - - `/var/vcap/jobs//monit`: The rendered monit file for that release job. (All such files are then gathered by the Bosh Agent in `/var/vcap/monit/job`, where the actual monit configuration relies, see below.) +- `/var/vcap/jobs/`: Contains evaluated release jobs for the instance. The Agent is responsible for managing which jobs are enabled or disabled on the VM. + - `/var/vcap/jobs//bin/`: Conventional location where Bosh renders the templates for wrapper scripts and [hook scripts](job-lifecycle.md). + - `/var/vcap/jobs//config/`: Conventional location for the job configuration files, like BPM config (in `/var/vcap/jobs/redis-server/config/bpm.yml`, responsible for starting executables). and other rendered config files (e.g. `/var/vcap/jobs/redis-server/config/redis.conf`). + - `/var/vcap/jobs//monit`: The rendered monit file for that release job. (All such files are then gathered by the Bosh Agent in `/var/vcap/monit/job`, where the actual monit configuration relies, see below.) --- + ## Storage Directories {: #storage } -* `/var/vcap/data/`: Directory that is used by the release jobs to keep _ephemeral_ data. Each release job usually creates a sub-folder with its name for namespacing (e.g. `redis-server` will place data into `/var/vcap/data/redis-server`). +- `/var/vcap/data/`: Directory that is used by the release jobs to keep _ephemeral_ data. Each release job usually creates a sub-folder with its name for namespacing (e.g. `redis-server` will place data into `/var/vcap/data/redis-server`). -* `/var/vcap/store/`: Directory that is used by the release jobs to keep _persistent_ data. Each release job usually creates a sub-folder with its name for namespacing (e.g. `redis-server` will place data into `/var/vcap/store/redis-server`). +- `/var/vcap/store/`: Directory that is used by the release jobs to keep _persistent_ data. Each release job usually creates a sub-folder with its name for namespacing (e.g. `redis-server` will place data into `/var/vcap/store/redis-server`). -* `/var/vcap/sys/run/`: Directory that is used by the release jobs to keep miscellaneous ephemeral data about currently running processes, for example, pid and lock files. Each release job usually creates a sub-folder with its name for namespacing (e.g. `redis-server` will place data into `/var/vcap/sys/run/redis-server`). +- `/var/vcap/sys/run/`: Directory that is used by the release jobs to keep miscellaneous ephemeral data about currently running processes, for example, pid and lock files. Each release job usually creates a sub-folder with its name for namespacing (e.g. `redis-server` will place data into `/var/vcap/sys/run/redis-server`). -* `/var/vcap/sys/log/`: Directory that is used by the release jobs to keep logs. Each release job usually creates a sub-folder with its name for namespacing (e.g. `redis-server` will place data into `/var/vcap/sys/log/redis-server`). Files in this directory are log rotated on a specific schedule configured by the Agent. +- `/var/vcap/sys/log/`: Directory that is used by the release jobs to keep logs. Each release job usually creates a sub-folder with its name for namespacing (e.g. `redis-server` will place data into `/var/vcap/sys/log/redis-server`). Files in this directory are log rotated on a specific schedule configured by the Agent. --- + ## Agent Configuration {: #agent } It's discouraged to modify or rely on the contents of this directory. -* `/var/vcap/bosh/`: Directory used by the Agent to keep its internal state. +- `/var/vcap/bosh/`: Directory used by the Agent to keep its internal state. -* `/var/vcap/bosh/agent.json`: Start up settings for the Agent that describe how to find bootstrap settings, and disable certain Agent functionality. +- `/var/vcap/bosh/agent.json`: Start up settings for the Agent that describe how to find bootstrap settings, and disable certain Agent functionality. -* `/var/vcap/bosh/settings.json`: Local copy of the bootstrap settings used by the Agent to configure network, system properties for the VM. They are refreshed every time Agent is restarted. +- `/var/vcap/bosh/settings.json`: Local copy of the bootstrap settings used by the Agent to configure network, system properties for the VM. They are refreshed every time Agent is restarted. -* `/var/vcap/bosh/update_settings.json`: The updated settings, as pushed by +- `/var/vcap/bosh/update_settings.json`: The updated settings, as pushed by the last `update_settings` RPC message sent through NATS to the Bosh Agent. These settings may differ from the bootstrap `settings.json`, especially with NATS client certificates, that are short-lived at bootstrap, and then replaced by the definitive client certificates, using `update_settings`. -* `/var/vcap/bosh/spec.json`: Instance settings used by the Agent to configure release jobs and packages for the VM. This file also includes structural info about the instance, like the deployment name (`deployment`), the instance group (`name`), the human-friendly instance ordinal (`index`), and the immutable instance UUID (`id`). These structural info are also put in the `/var/vcap/instance` directory for easier access, see [Instance Metadata on Filesystem](instance-metadata.md#fs) for more details. +- `/var/vcap/bosh/spec.json`: Instance settings used by the Agent to configure release jobs and packages for the VM. This file also includes structural info about the instance, like the deployment name (`deployment`), the instance group (`name`), the human-friendly instance ordinal (`index`), and the immutable instance UUID (`id`). These structural info are also put in the `/var/vcap/instance` directory for easier access, see [Instance Metadata on Filesystem](instance-metadata.md#fs) for more details. -* `/var/vcap/bosh/log/current`: Current Agent log. Agent's logs are logrotated and archives are kept in `/var/vcap/bosh/log/` directory. +- `/var/vcap/bosh/log/current`: Current Agent log. Agent's logs are logrotated and archives are kept in `/var/vcap/bosh/log/` directory. -* `/var/vcap/bosh/etc/ntpserver`: File with a list of NTP servers configured by the Agent. This file is used by `/var/vcap/bosh/bin/ntpdate` to keep time in sync. +- `/var/vcap/bosh/etc/ntpserver`: File with a list of NTP servers configured by the Agent. This file is used by `/var/vcap/bosh/bin/ntpdate` to keep time in sync. --- + ## Monit Configuration {: #monit } It's discouraged to modify or rely on the contents of this directory. -* `/var/vcap/monit/job`: Directory that keeps current Monit configuration files. The Agent is responsible for updating files in this directory when deployment job is updated. +- `/var/vcap/monit/job`: Directory that keeps current Monit configuration files. The Agent is responsible for updating files in this directory when deployment job is updated. -* `/var/vcap/monit/monit.log`: Monit activity log. Includes information about starts, stops, restarts, etc. of release job processes monitored by Monit. +- `/var/vcap/monit/monit.log`: Monit activity log. Includes information about starts, stops, restarts, etc. of release job processes monitored by Monit. [1] “VCAP” stands for “VMware Cloud Application Platform”. diff --git a/content/vm-monit.md b/content/vm-monit.md index 54ee0dd52..56425fd71 100644 --- a/content/vm-monit.md +++ b/content/vm-monit.md @@ -1,5 +1,9 @@ +# Process Monitoring with Monit + The Agent on each deployment job VM is responsible for managing lifecycle of each enabled release job. It starts, monitors, restarts and stops release jobs' processes. These tasks are done with the help of [Monit, version 5.2.5](https://web.archive.org/web/20110816041503/https://mmonit.com/monit/documentation/monit.html). The Agent communicates with the Monit daemon through Monit HTTP APIs to add, remove, start, stop, monitor and unmonitor release jobs' processes. + --- + ## Check Status {: #check-status } Assuming you have a deployment, run `bosh instances` to see aggregate status for each deployment job VM: @@ -10,7 +14,7 @@ bosh instances Should result in: -```text +```shell Deployment `my-deployment' @@ -40,6 +44,7 @@ There are several typical values for State: To determine what the problem is with a specific VM, you can ssh into the VM and look at the logs and/or Monit directly. --- + ## Using Monit on the VM {: #using-monit } On any BOSH-managed VM, you can access Monit status for release jobs' @@ -60,7 +65,7 @@ monit summary Should result in: -```text +```shell The Monit daemon 5.2.4 uptime: 1d 22h 7m Process 'nats' running @@ -79,7 +84,7 @@ System 'system_bm-24638eb6-55b9-4670-bb1a-23c9e3f77d91' running ``` !!! note - You can use standard watch utility with the summary command to track process status over time. + You can use standard `watch` utility with the summary command to track process status over time. You can also get more detailed information about individual processes via `monit status`: @@ -89,7 +94,7 @@ monit status Should result in: -```text +```shell The Monit daemon 5.2.4 uptime: 1d 22h 8m Process 'nats' diff --git a/content/vm-struct.md b/content/vm-struct.md index 8749d4399..4eb5a3001 100644 --- a/content/vm-struct.md +++ b/content/vm-struct.md @@ -1,3 +1,5 @@ +# Structure of a VM + All managed VMs include: - BOSH Agent diff --git a/content/vsphere-cpi-errors.md b/content/vsphere-cpi-errors.md index fc2587f03..c78aab9f4 100644 --- a/content/vsphere-cpi-errors.md +++ b/content/vsphere-cpi-errors.md @@ -1,6 +1,10 @@ +# vSphere CPI Errors + ## Field object_set is not optional -> Field object_set is not optional +```shell +Field object_set is not optional +``` This error may occur when global CPI configuration references: @@ -8,31 +12,35 @@ This error may occur when global CPI configuration references: - stemcell that cannot be found in the templates folder - cluster (via `clusters` key) that does not have any hosts (as a consequence of having no hosts , it has access to zero datastores) - ## Missing Properties Exception -> ...should have the following properties: ["info.progress", "info.state", "info.result", "info.error"] (VSphereCloud::CloudSearcher::MissingPropertiesException), but they were missing these: # +```shell +...should have the following properties: ["info.progress", "info.state", "info.result", "info.error"] (VSphereCloud::CloudSearcher::MissingPropertiesException), but they were missing these: # +``` Add `System.View` on vCenter server level so that persistent disks can be moved between the datastores. - ## Field counter_id is not optional -> Field counter_id is not optional -> ...lib/cloud/vsphere/client.rb:270:in `fetch_perf_metric_names' +```shell +Field counter_id is not optional +...lib/cloud/vsphere/client.rb:270:in `fetch_perf_metric_names' +``` The CPI requires access to performance metrics from ESXi hosts. This error may be returned if one of the hosts in the cluster is not returning these metrics (e.g. `memory.usage.average`). Possible solution is to [restart management agents](http://www.running-system.com/no-cpu-and-memory-usage-data-from-host-available-in-vcenter/) on the hosts. - ## Failed to add disk -> Failed to add disk scsi0:2. +```shell +Failed to add disk scsi0:2. +``` This error typically occurs when persistent disk is being attached to a second VM while it is attached to another VM. That may happen when first VM was not properly deleted and BOSH is no longer aware of its existence. - ## Could not acquire HTTP NFC lease -> Could not acquire HTTP NFC lease, message is: 'A specified parameter was not correct.' fault cause is: '', fault message is: [], dynamic type is '', dynamic property is []' +```shell +Could not acquire HTTP NFC lease, message is: 'A specified parameter was not correct.' fault cause is: '', fault message is: [], dynamic type is '', dynamic property is []' +``` The [vCenter docs](https://www.vmware.com/support/developer/vc-sdk/visdk41pubs/ApiReference/vim.vm.DefaultPowerOpInfo.html) show that the value should be `preset` rather than `default` inside the OVF file. Switching `powerOpInfo.*` properties resolved the problem. diff --git a/content/vsphere-cpi.md b/content/vsphere-cpi.md index 0a882c183..9d4644688 100644 --- a/content/vsphere-cpi.md +++ b/content/vsphere-cpi.md @@ -1,20 +1,22 @@ -This topic describes cloud properties supported for different resources created by the vSphere CPI. The cloud properties can be specified at the different levels supported in the [Cloud Config](cloud-config.md). Typically, define defaults at the global level and override them at the [`azs`](cloud-config.md#azs), [`networks`](cloud-config.md#networks), [`vm_types`](cloud-config.md#vm-types) [`vm_extension`](cloud-config.md#vm-extensions) or [`disk_types`](#disk-types) levels. +# VMware vSphere + +This topic describes cloud properties supported for different resources created by the vSphere CPI. The cloud properties can be specified at the different levels supported in the [Cloud Config](cloud-config.md). Typically, define defaults at the global level and override them at the [`azs`](cloud-config.md#azs), [`networks`](cloud-config.md#networks), [`vm_types`](cloud-config.md#vm-types) [`vm_extension`](cloud-config.md#vm-extensions) or [`disk_types`](#disk-types) levels. ## AZs {: #azs } Schema for `cloud_properties` section: -* **datacenters** [Array, optional]: Array of datacenters to use for VM placement. Must have only one and it must match datacenter configured in global CPI options. - * **name** [String, required]: Datacenter name. - * **clusters** [Array, required]: Array of clusters to use for VM placement. - * **<cluster name>** [String, required]: Cluster name. - * **resource_pool** [String, optional]: Name of vSphere Resource Pool to use for VM placement. - * **host_group** [Dictionary, optional]: Properties of the Host Group to use for VM placement. Available in v52+. (Backwards compatible with old format of specifying Host Group with just a name string of the host group ) - * **name** [String, required]: Name of the host group in vSphere - * **drs_rule** [String, optional]: One of the values from MUST or SHOULD. Is case insensitive. Defaults to SHOULD if not specified or in case of a spelling mistake. - * **drs_rules** [Array, optional]: Array of DRS rules applied to [constrain VM placement](vm-anti-affinity.md#vsphere). Must have only one. - * **name** [String, required]: Name of a DRS rule that the Director will create. - * **type** [String, required]: Type of a DRS rule. Currently only `separate_vms` is supported. +- **datacenters** [Array, optional]: Array of datacenters to use for VM placement. Must have only one and it must match datacenter configured in global CPI options. + - **name** [String, required]: Datacenter name. + - **clusters** [Array, required]: Array of clusters to use for VM placement. + - **<cluster name>** [String, required]: Cluster name. + - **resource_pool** [String, optional]: Name of vSphere Resource Pool to use for VM placement. + - **host_group** [Dictionary, optional]: Properties of the Host Group to use for VM placement. Available in v52+. (Backwards compatible with old format of specifying Host Group with just a name string of the host group ) + - **name** [String, required]: Name of the host group in vSphere + - **drs_rule** [String, optional]: One of the values from MUST or SHOULD. Is case insensitive. Defaults to SHOULD if not specified or in case of a spelling mistake. + - **drs_rules** [Array, optional]: Array of DRS rules applied to [constrain VM placement](vm-anti-affinity.md#vsphere). Must have only one. + - **name** [String, required]: Name of a DRS rule that the Director will create. + - **type** [String, required]: Type of a DRS rule. Currently only `separate_vms` is supported. Example: @@ -41,12 +43,12 @@ azs: ``` --- + ## Networks {: #networks } Schema for `cloud_properties` section used by manual network subnet: -* **name** [String, required]: Name of the vSphere network. Example: `VM Network`. - +- **name** [String, required]: Name of the vSphere network. Example: `VM Network`. Example of manual network: @@ -64,46 +66,47 @@ networks: vSphere CPI does not support dynamic and vip networks. --- + ## VM Types / VM Extensions {: #resource-pools } Schema for `cloud_properties` section: -* **cpu** [Integer, required]: Number of CPUs. Example: `1`. -* **ram** [Integer, required]: RAM in megabytes. Example: `1024`. -* **disk** [Integer, required]: Ephemeral disk size in megabytes. Example: `10240`. -* **cpu\_hot\_add\_enabled** [Boolean, optional]: Allows operator to add additional CPU resources while the VM is on. Default: `false`. Available in v21+. -* **cpu\_reserve\_full\_mhz** [Boolean, optional]: If set true, CPU resource reservation for this virtual machine will always be equal to the max MHz of the ESXi host times the number of CPUs requested for the VM. Default: `false`. Available in v97.0.15+. -* **memory\_reservation\_locked\_to\_max** [Boolean, optional]: If set true, memory resource reservation for this virtual machine will always be equal to the virtual machine's memory size. Default: `false`. Available in v82+. -* **memory\_hot\_add\_enabled** [Boolean, optional]: Allows operator to add additional memory resources while the VM is on. Default: `false`. Available in v21+. -* **upgrade\_hw\_version** [Boolean, optional]: Upgrades the virtual hardware version of a virtual machine to the latest supported version on the ESXi host. Overrides the global upgrade_hw_version. Default: `false`. -* **nested\_hardware\_virtualization** [Boolean, optional]: Exposes hardware assisted virtualization to the VM. Default: `false`. -* **enable\_human\_readable\_name** [Boolean, optional]: Use a name generated by instance group name and deployment name for the VM. Default: `false`. Available in v53+. -* **datastores** [Array, optional]: Allows operator to specify a list of ephemeral datastores, datastore clusters for the VM. Datastore names are exact datastore names and not regex patterns. At least one of these datastores must be accessible from clusters provided in `resource_pools.cloud_properties`/`azs.cloud_properties` or in the global CPI configuration. Available in v23+. Datastore Clusters can be specified as a hash in format of `{clusters: [ datastoreCluster1: {}, datastoreCluster2: {}]}`. Clusters whose Storage DRS is turned off will be ignored. -* **datacenters** [Array, optional]: Used to override the VM placement specified under `azs.cloud_properties`. The format is the same as under [`AZs`](#azs). -* **tags** [Array, optional]: A list of category and tag name-value pairs used to attach to created VMs. Available in v53+. Available on vCenter 6.5+. The tags to be attached must be exist on vCenter host. For each tag to be attached, both category and tag names should be specified. **Note:** This `tags` property is unrelated to the top-level `tags` block in [runtime](runtime-config/#tags) and [deployment](manifest-v2/#tags) configs. -* **vm_group** [String, optional]: Name of VM Group this VM should be part of. -* **disable\_drs** [Boolean, Optional]: Disables DRS on this VM type. In short, VM created with this vm_type will *NOT* be v-motioned by DRS. Available in v53+ -* **nsx** [Dictionary, optional]: [VMware NSX](http://www.vmware.com/products/nsx.html) additions section. Available in CPI v30+ and NSX v6.1+. - * **security_groups** [Array, optional]: A collection of [security group](https://pubs.vmware.com/NSX-6/index.jsp#com.vmware.nsx.admin.doc/GUID-16B3134E-DDF1-445A-8646-BB0E98C3C9B5.html) names that the instances should belong to. The CPI will create the security groups if they do not exist. +- **cpu** [Integer, required]: Number of CPUs. Example: `1`. +- **ram** [Integer, required]: RAM in megabytes. Example: `1024`. +- **disk** [Integer, required]: Ephemeral disk size in megabytes. Example: `10240`. +- **cpu\_hot\_add\_enabled** [Boolean, optional]: Allows operator to add additional CPU resources while the VM is on. Default: `false`. Available in v21+. +- **cpu\_reserve\_full\_mhz** [Boolean, optional]: If set true, CPU resource reservation for this virtual machine will always be equal to the max MHz of the ESXi host times the number of CPUs requested for the VM. Default: `false`. Available in v97.0.15+. +- **memory\_reservation\_locked\_to\_max** [Boolean, optional]: If set true, memory resource reservation for this virtual machine will always be equal to the virtual machine's memory size. Default: `false`. Available in v82+. +- **memory\_hot\_add\_enabled** [Boolean, optional]: Allows operator to add additional memory resources while the VM is on. Default: `false`. Available in v21+. +- **upgrade\_hw\_version** [Boolean, optional]: Upgrades the virtual hardware version of a virtual machine to the latest supported version on the ESXi host. Overrides the global upgrade_hw_version. Default: `false`. +- **nested\_hardware\_virtualization** [Boolean, optional]: Exposes hardware assisted virtualization to the VM. Default: `false`. +- **enable\_human\_readable\_name** [Boolean, optional]: Use a name generated by instance group name and deployment name for the VM. Default: `false`. Available in v53+. +- **datastores** [Array, optional]: Allows operator to specify a list of ephemeral datastores, datastore clusters for the VM. Datastore names are exact datastore names and not regex patterns. At least one of these datastores must be accessible from clusters provided in `resource_pools.cloud_properties`/`azs.cloud_properties` or in the global CPI configuration. Available in v23+. Datastore Clusters can be specified as a hash in format of `{clusters: [ datastoreCluster1: {}, datastoreCluster2: {}]}`. Clusters whose Storage DRS is turned off will be ignored. +- **datacenters** [Array, optional]: Used to override the VM placement specified under `azs.cloud_properties`. The format is the same as under [`AZs`](#azs). +- **tags** [Array, optional]: A list of category and tag name-value pairs used to attach to created VMs. Available in v53+. Available on vCenter 6.5+. The tags to be attached must be exist on vCenter host. For each tag to be attached, both category and tag names should be specified. **Note:** This `tags` property is unrelated to the top-level `tags` block in [runtime](runtime-config/#tags) and [deployment](manifest-v2/#tags) configs. +- **vm_group** [String, optional]: Name of VM Group this VM should be part of. +- **disable\_drs** [Boolean, Optional]: Disables DRS on this VM type. In short, VM created with this vm_type will *NOT* be v-motioned by DRS. Available in v53+ +- **nsx** [Dictionary, optional]: [VMware NSX](http://www.vmware.com/products/nsx.html) additions section. Available in CPI v30+ and NSX v6.1+. + - **security_groups** [Array, optional]: A collection of [security group](https://pubs.vmware.com/NSX-6/index.jsp#com.vmware.nsx.admin.doc/GUID-16B3134E-DDF1-445A-8646-BB0E98C3C9B5.html) names that the instances should belong to. The CPI will create the security groups if they do not exist. BOSH will also automatically create security groups based on metadata such as deployment name and instance group name. The full list of groups can be seen under [create_vm's environment groups](cpi-api-v2.md#create-vm). **The security groups, if specified under NSX load balancers need not be specified(duplicated) again here. Although CPI makes best effort to de-duplicate them, it is advised not to specify them again.** - * **lbs** [Array, optional]: A collection of [NSX Edge Load Balancers](https://pubs.vmware.com/NSX-6/index.jsp?topic=%2Fcom.vmware.nsx.admin.doc%2FGUID-152982CF-108F-47A6-B86A-0F0F6A56D628.html) (LBs) to which instances should be attached. The LB and [Server Pool](https://pubs.vmware.com/NSX-6/index.jsp?topic=%2Fcom.vmware.nsx.admin.doc%2FGUID-D5A3BDBA-57A6-43F4-AE5E-3A387FE69EDC.html) must exist prior to the deployment. - * **edge_name** [String, required]: Name of the NSX Edge. - * **pool_name** [String, required]: Name of the Edge's Server Pool. - * **security_group** [String, required]: Name of the Pool's target Security Group. The CPI will add the VM to the specified security group (creating the security group if needed), then add the security group to the specified Server Pool. - * **port** [Integer, required]: The port that the VM's service is listening on (e.g. 80 for HTTP). - * **monitor_port** [Integer, optional]: The healthcheck port that the VM is listening on. Defaults to the value of `port`. -* **vmx_options** [Dictionary, optional]: Allows operator to specify [VM advanced configuration options](https://docs.vmware.com/en/VMware-vSphere/6.0/com.vmware.vsphere.resmgmt.doc/GUID-F8C7EF4D-D023-4F54-A2AB-8CF840C10939.html). All values are subject to YAML's type interpretation, and given that for certain configuration options vSphere will accept only a specific value type please take note of the difference between values with similar appearances such as: `true` vs `"true"` and `"1234"` vs `1234`. Refer to the vSphere documentation for more information about what configuration options are accepted. Available in v42+. -* **storage\_policy** [Dictionary, optional]: Storage Policy which is applied to a VM and its ephemeral disk. Available in v53+ - * **name** [String, optional]: Name of the storage policy to be applied to the VM and its ephemeral disk. Available in v53+ -* **nsxt** [Dictionary, optional]: [VMware NSX](http://www.vmware.com/products/nsx.html) additions section. Available in CPI v45+. - * **ns_groups** [Array, optional]: A collection of [NS Groups](http://pubs.vmware.com/nsxt-11/index.jsp?topic=%2Fcom.vmware.nsxt.admin.doc%2FGUID-718E769B-8D89-485B-8DBD-04F1F82CFE14.html) names that the instances should belong to. Available in NSX-T v1.1+. - * **vif_type** [String, optional]: Supported types: `PARENT`, `null`. Overrides the global `default_vif_type`. Available in NSX-T v2.0+. - * **lb** [Dictionary, optional]: NSX-T logical Load Balancer. Available in CPI v48+ - * **server_pools** [Array, optional] Server Pool must exist prior to the deployment. For static server pool, VM is directly added to the server pool. If server pool is dynamic, CPI looks up the NSGroup and adds the VM to the NSGroup. - * **name** [String, required]: Name of the Server Pool - * **port** [Integer, optional]: The port that the VM's service is listening on (e.g. 80 for HTTP). If port is specified, all connections will be sent to this port on the VM. Only specify a single port (no ranges). If unset, the load balancer will connect the client to the VM using the same port number (e.g. if the client connects to port 443, the load balancer will forward to the VM on port 443). - * **tag_nsx_vm_objects** [Boolean, optional]: When enabled, tag NSX VM objects with the same set of tags as vsphere VM objects. Available in CPI v97.0.14+. + - **lbs** [Array, optional]: A collection of [NSX Edge Load Balancers](https://pubs.vmware.com/NSX-6/index.jsp?topic=%2Fcom.vmware.nsx.admin.doc%2FGUID-152982CF-108F-47A6-B86A-0F0F6A56D628.html) (LBs) to which instances should be attached. The LB and [Server Pool](https://pubs.vmware.com/NSX-6/index.jsp?topic=%2Fcom.vmware.nsx.admin.doc%2FGUID-D5A3BDBA-57A6-43F4-AE5E-3A387FE69EDC.html) must exist prior to the deployment. + - **edge_name** [String, required]: Name of the NSX Edge. + - **pool_name** [String, required]: Name of the Edge's Server Pool. + - **security_group** [String, required]: Name of the Pool's target Security Group. The CPI will add the VM to the specified security group (creating the security group if needed), then add the security group to the specified Server Pool. + - **port** [Integer, required]: The port that the VM's service is listening on (e.g. 80 for HTTP). + - **monitor_port** [Integer, optional]: The healthcheck port that the VM is listening on. Defaults to the value of `port`. +- **vmx_options** [Dictionary, optional]: Allows operator to specify [VM advanced configuration options](https://docs.vmware.com/en/VMware-vSphere/6.0/com.vmware.vsphere.resmgmt.doc/GUID-F8C7EF4D-D023-4F54-A2AB-8CF840C10939.html). All values are subject to YAML's type interpretation, and given that for certain configuration options vSphere will accept only a specific value type please take note of the difference between values with similar appearances such as: `true` vs `"true"` and `"1234"` vs `1234`. Refer to the vSphere documentation for more information about what configuration options are accepted. Available in v42+. +- **storage\_policy** [Dictionary, optional]: Storage Policy which is applied to a VM and its ephemeral disk. Available in v53+ + - **name** [String, optional]: Name of the storage policy to be applied to the VM and its ephemeral disk. Available in v53+ +- **nsxt** [Dictionary, optional]: [VMware NSX](http://www.vmware.com/products/nsx.html) additions section. Available in CPI v45+. + - **ns_groups** [Array, optional]: A collection of [NS Groups](http://pubs.vmware.com/nsxt-11/index.jsp?topic=%2Fcom.vmware.nsxt.admin.doc%2FGUID-718E769B-8D89-485B-8DBD-04F1F82CFE14.html) names that the instances should belong to. Available in NSX-T v1.1+. + - **vif_type** [String, optional]: Supported types: `PARENT`, `null`. Overrides the global `default_vif_type`. Available in NSX-T v2.0+. + - **lb** [Dictionary, optional]: NSX-T logical Load Balancer. Available in CPI v48+ + - **server_pools** [Array, optional] Server Pool must exist prior to the deployment. For static server pool, VM is directly added to the server pool. If server pool is dynamic, CPI looks up the NSGroup and adds the VM to the NSGroup. + - **name** [String, required]: Name of the Server Pool + - **port** [Integer, optional]: The port that the VM's service is listening on (e.g. 80 for HTTP). If port is specified, all connections will be sent to this port on the VM. Only specify a single port (no ranges). If unset, the load balancer will connect the client to the VM using the same port number (e.g. if the client connects to port 443, the load balancer will forward to the VM on port 443). + - **tag_nsx_vm_objects** [Boolean, optional]: When enabled, tag NSX VM objects with the same set of tags as vsphere VM objects. Available in CPI v97.0.14+. - **pci_passthroughs** [Array, optional]: Specifies a PCI (Peripheral Component Interconnect) device to attach to VM via vSphere Dynamic DirectPath IO. Requires vSphere 7.0+. Automatically sets the properties `memory_reservation_locked_to_max` and `upgrade_hw_version` to `true`. Each entry requires the PCI card's `device_id` and `vendor_id`. Available in v97+. - **vgpus** [Array, optional]: Specifies an Nvidia GRID vGPU to attach to VM. Automatically sets the properties `memory_reservation_locked_to_max` and `upgrade_hw_version` to `true`. Available in v97+. - **device_groups** [Array, optional]: Specifies Device Groups to attach to the VM. Requires vSphere 8.0+. Supports groups of Nvidia vGPU devices. Device groups must be configured in vSphere. Available in v98.0.2+. @@ -211,16 +214,17 @@ vm_extensions: ``` --- + ## Disk Types {: #disk-pools } Schema for `cloud_properties` section: -* **type** [String, optional]: +- **type** [String, optional]: [Virtual disk type](http://pubs.vmware.com/vi-sdk/visdk250/ReferenceGuide/vim.VirtualDiskManager.VirtualDiskType.html) used for persistent disks: `thick`, `thin`, `preallocated`, `eagerZeroedThick`. Defaults to `preallocated`. Available in v12. Overrides the global `default_disk_type`. -* **datastores** [Array, optional]: List of datastore names, datastore clusters for storing persistent disks. Overrides the global `persistent_datastore_pattern`. These names are exact datastore names and not regex patterns. Available in v29+. Datastore Clusters can be specified as a hash in format of `{clusters: [ datastoreCluster1: {}, datastoreCluster2: {}]}`. Clusters whose Storage DRS is turned off will be ignored. +- **datastores** [Array, optional]: List of datastore names, datastore clusters for storing persistent disks. Overrides the global `persistent_datastore_pattern`. These names are exact datastore names and not regex patterns. Available in v29+. Datastore Clusters can be specified as a hash in format of `{clusters: [ datastoreCluster1: {}, datastoreCluster2: {}]}`. Clusters whose Storage DRS is turned off will be ignored. Example of 10GB disk: @@ -254,60 +258,61 @@ Example of persistent disk stored in specific datastores: ``` --- + ## Global Configuration {: #global } The CPI can only talk to a single vCenter installation and manage VMs within a single vSphere datacenter. Schema: -* **host** [String, required]: IP address or hostname of vCenter. Example: `172.16.68.3`. -* **user** [String, required]: Username for the API access. Example: `root`. -* **password** [String, required]: Password for the API access. Example: `vmware` -* **connection_options** [Object, optional]: Additional connection options - * **ca_cert** [String, optional]: A list of concatenated CA certificates used to verify the TLS connection to the vCenter server. If no value is provided, the CPI will establish a TLS connection but will not verify the certificate presented by the server. -* **http_logging** [Boolean, optional]: Enables logging all HTTP requests and responses to vSphere API. Default: `false`. Available in v37+. -* **default\_disk\_type** [String, optional]: Sets the default +- **host** [String, required]: IP address or hostname of vCenter. Example: `172.16.68.3`. +- **user** [String, required]: Username for the API access. Example: `root`. +- **password** [String, required]: Password for the API access. Example: `vmware` +- **connection_options** [Object, optional]: Additional connection options + - **ca_cert** [String, optional]: A list of concatenated CA certificates used to verify the TLS connection to the vCenter server. If no value is provided, the CPI will establish a TLS connection but will not verify the certificate presented by the server. +- **http_logging** [Boolean, optional]: Enables logging all HTTP requests and responses to vSphere API. Default: `false`. Available in v37+. +- **default\_disk\_type** [String, optional]: Sets the default [disk type](https://www.vmware.com/support/developer/converter-sdk/conv51_apireference/vim.VirtualDiskManager.VirtualDiskType.html). Can be either `thin` or `preallocated`, defaults to `preallocated`. `preallocated` sets "all space allocated at [VM] creation time and the space is zeroed on demand as the space is used", and `thin`, "virtual disk is allocated and zeroed on demand as the space is used." Applies to both root and ephemeral. May also apply to persistent disks unless overridden in [disk pool](#disk-types--disk-pools-). -* **default\_scsi\_controller\_type** [String, optional]: SCSI controller type for VMs. Can be `paravirtual` (supports up to 63 disks), `lsi_logic` (preserves stemcell controller, may reduce write latency), or `lsi_logic_sas`. Default: `paravirtual`. Available in v98.0.5+. -* **ensure_no_ip_conflicts** [Boolean, optional]: When creating a VM, ensure that no other VMs exist in the same port group with the same IP address. The CPI queries the vCenter to detect conflict, does not use `ping`. Default: `true`. Available in v97.0.5+. - -* **datacenters** [Array, optional]: Array of datacenters to use for VM placement. Must have only one. - * **name** [String, required]: Datacenter name. - * **vm_folder** [String, optional]: Path to a folder (relative to the datacenter) for storing created VMs. Folder will be automatically created if not found. Defaults to `BOSH_VMs`. - * **template_folder** [String, optional]: Path to a folder (relative to the datacenter) for storing uploaded stemcells. Folder will be automatically created if not found. Defaults to `BOSH_Templates`. - * **disk_path** [String, optional]: Path to a *disk* folder for storing persistent disks. Folder will be automatically created in the datastore if not found. Defaults to `BOSH_Disks`. - * **datastore_pattern** [String, required if `datastore_cluster_pattern` is not set or CPI version <= v63]: Pattern for selecting datastores for storing ephemeral disks and stemcells. - * **datastore\_cluster\_pattern** [String, required if `datastore_pattern` is not set]: Pattern for selecting datastore clusters for storing ephemeral disks. Clusters whose Storage DRS is turned off will be ignored. As of v94, datastore clusters in folders can be referenced using the full path with slashes as the delimiter. - * **persistent\_datastore\_pattern** [String, required if `persistent_datastore_cluster_pattern` is not set]: Pattern for selecting datastores for storing persistent disks. - * **persistent\_datastore\_cluster\_pattern** [String, required if `persistent_datastore_pattern` is not set]: Pattern for selecting datastore clusters for storing persistent disks. Clusters whose Storage DRS is turned off will be ignored. As of v94, datastore clusters in folders can be referenced using the full path with slashes as the delimiter. - * **clusters** [Array, required]: Array of clusters to use for VM placement. - * **<cluster name>** [String, required]: Cluster name. - * **resource_pool** [String, optional]: Specific vSphere resource pool to use within the cluster. - * **host_group** [String, optional]: Specific Host Group to use for VM placement. Available in v52+. -* **nsx** [Dictionary, optional]: NSX-V configuration options. This is required if the other NSX features are used below (e.g. 'security_groups' for `resource_pools`). - * **address** [String, required]: The NSX server's address. Can be a hostname (e.g. `nsx-server.example.com`) or an IP address. - * **user** [String, required]: The login username for the NSX server. - * **password** [String, required]: The login password for the NSX server. - * **ca_cert** [String, optional]: A CA certificate that can authenticate the NSX server certificate. **Required** if the NSX Manager has a self-signed SSL certificate. Must be in PEM format. -* **enable\_auto\_anti\_affinity\_drs\_rules** [Boolean, optional]: Creates DRS rule to place VMs on separate hosts. DRS Automation Level must be set to "Fully Automated"; does not work when DRS is set to "Partially Automated" or "Manual". May cause VMs to fail to power on if there are more VMs than hosts after initial deployment. Default: `false`. Available in v33+. -* **vm\_storage\_policy\_name** [Boolean, optional]: Name of the storage Policy which is applied to a VM and its ephemeral disk. Available v53+ -* **upgrade\_hw\_version** [Boolean, optional]: Upgrades the virtual hardware version of a virtual machine to the latest supported version on the ESXi host. Default: `false`. -* **nsxt** [Dictionary, optional]: NSX-T configuration options. Available in v45+. - * **use\_policy\_api** [Boolean, optional]: Enabling this feature will use the [NSX-T Policy API](https://blogs.vmware.com/networkvirtualization/2020/06/navigating-nsxt-policy-apis.html/) instead of the Manager API. It affects the attachment of VMs to NS Groups and VM placement in static Load Balancer Pools. This feature requires NSX-T Data Center v3.0 or later. Default: false. Available in v56+. In v58+, the VM's NSX-T segment ports are also tagged. The tags are prepended with `bosh/` and include the key/value pairs specified in the tags blocks of the [deployment](manifest-v2/#tags) or [runtime](runtime-config/#tags) configurations as well as BOSH default metadata (`instance_group`, `job`, `index`, etc.) +- **default\_scsi\_controller\_type** [String, optional]: SCSI controller type for VMs. Can be `paravirtual` (supports up to 63 disks), `lsi_logic` (preserves stemcell controller, may reduce write latency), or `lsi_logic_sas`. Default: `paravirtual`. Available in v98.0.5+. +- **ensure_no_ip_conflicts** [Boolean, optional]: When creating a VM, ensure that no other VMs exist in the same port group with the same IP address. The CPI queries the vCenter to detect conflict, does not use `ping`. Default: `true`. Available in v97.0.5+. + +- **datacenters** [Array, optional]: Array of datacenters to use for VM placement. Must have only one. + - **name** [String, required]: Datacenter name. + - **vm_folder** [String, optional]: Path to a folder (relative to the datacenter) for storing created VMs. Folder will be automatically created if not found. Defaults to `BOSH_VMs`. + - **template_folder** [String, optional]: Path to a folder (relative to the datacenter) for storing uploaded stemcells. Folder will be automatically created if not found. Defaults to `BOSH_Templates`. + - **disk_path** [String, optional]: Path to a *disk* folder for storing persistent disks. Folder will be automatically created in the datastore if not found. Defaults to `BOSH_Disks`. + - **datastore_pattern** [String, required if `datastore_cluster_pattern` is not set or CPI version <= v63]: Pattern for selecting datastores for storing ephemeral disks and stemcells. + - **datastore\_cluster\_pattern** [String, required if `datastore_pattern` is not set]: Pattern for selecting datastore clusters for storing ephemeral disks. Clusters whose Storage DRS is turned off will be ignored. As of v94, datastore clusters in folders can be referenced using the full path with slashes as the delimiter. + - **persistent\_datastore\_pattern** [String, required if `persistent_datastore_cluster_pattern` is not set]: Pattern for selecting datastores for storing persistent disks. + - **persistent\_datastore\_cluster\_pattern** [String, required if `persistent_datastore_pattern` is not set]: Pattern for selecting datastore clusters for storing persistent disks. Clusters whose Storage DRS is turned off will be ignored. As of v94, datastore clusters in folders can be referenced using the full path with slashes as the delimiter. + - **clusters** [Array, required]: Array of clusters to use for VM placement. + - **<cluster name>** [String, required]: Cluster name. + - **resource_pool** [String, optional]: Specific vSphere resource pool to use within the cluster. + - **host_group** [String, optional]: Specific Host Group to use for VM placement. Available in v52+. +- **nsx** [Dictionary, optional]: NSX-V configuration options. This is required if the other NSX features are used below (e.g. 'security_groups' for `resource_pools`). + - **address** [String, required]: The NSX server's address. Can be a hostname (e.g. `nsx-server.example.com`) or an IP address. + - **user** [String, required]: The login username for the NSX server. + - **password** [String, required]: The login password for the NSX server. + - **ca_cert** [String, optional]: A CA certificate that can authenticate the NSX server certificate. **Required** if the NSX Manager has a self-signed SSL certificate. Must be in PEM format. +- **enable\_auto\_anti\_affinity\_drs\_rules** [Boolean, optional]: Creates DRS rule to place VMs on separate hosts. DRS Automation Level must be set to "Fully Automated"; does not work when DRS is set to "Partially Automated" or "Manual". May cause VMs to fail to power on if there are more VMs than hosts after initial deployment. Default: `false`. Available in v33+. +- **vm\_storage\_policy\_name** [Boolean, optional]: Name of the storage Policy which is applied to a VM and its ephemeral disk. Available v53+ +- **upgrade\_hw\_version** [Boolean, optional]: Upgrades the virtual hardware version of a virtual machine to the latest supported version on the ESXi host. Default: `false`. +- **nsxt** [Dictionary, optional]: NSX-T configuration options. Available in v45+. + - **use\_policy\_api** [Boolean, optional]: Enabling this feature will use the [NSX-T Policy API](https://blogs.vmware.com/networkvirtualization/2020/06/navigating-nsxt-policy-apis.html/) instead of the Manager API. It affects the attachment of VMs to NS Groups and VM placement in static Load Balancer Pools. This feature requires NSX-T Data Center v3.0 or later. Default: false. Available in v56+. In v58+, the VM's NSX-T segment ports are also tagged. The tags are prepended with `bosh/` and include the key/value pairs specified in the tags blocks of the [deployment](manifest-v2/#tags) or [runtime](runtime-config/#tags) configurations as well as BOSH default metadata (`instance_group`, `job`, `index`, etc.) * **policy\_api\_migration\_mode** [Boolean, optional] This option requires `use_policy_api` to be set to `true`. When enabled, the CPI attempts to associate VMs in both the Policy API and the Manager API. The VM is associated with groups and server pools in the Policy API, and with NSGroups and server pools in the Manager API. It will return an error if the Manager API objects do not exist, but not if the Policy API objects do not exist. This option is only intended to be used in conjunction with scripts to help migrate NSX-T entities from the Manager API to the Policy API. Default: `false`. Available in v74+. - * **host** [String, required]: The NSX-T server's address. Can be a hostname (e.g. `nsx-server.example.com`) or an IP address. - * **username** [String, required]: The login username for the NSX-T server. - * **password** [String, required]: The login password for the NSX-T server. - * **ca_cert** [String, optional]: A CA certificate that can authenticate the NSX-T server certificate. **Required** if the NSX-T Manager has a self-signed SSL certificate. Must be in PEM format. - * **default_vif_type** [String, optional]: Supported Types: `PARENT`. Default VIF type attached to logical port. Available in NSX-T v2.0+. - * **auth_certificate** [String, optional]: Certificate used for certificate-based authentication. Certificate-based authentication takes precedence over username/password if both are specified. Available in v51+. - * **auth_private_key** [String, optional]: Private key file used for certificate-based authentication. Available in v51+. - * **remote_auth** [Boolean, optional]: Enables remote authentication for NSX-T via vIDM. Available in v52.1.5+ and v53.0.1+ - * **allow_overwrite** [Boolean, optional]: When enabled, the CPI sets the `X-Allow-Overwrite` header to 'true' when making NSX-T Management API requests, which allows the Management API to mutate Policy API objects. Default: `true` for backwards compatibility. Available in v91+. + - **host** [String, required]: The NSX-T server's address. Can be a hostname (e.g. `nsx-server.example.com`) or an IP address. + - **username** [String, required]: The login username for the NSX-T server. + - **password** [String, required]: The login password for the NSX-T server. + - **ca_cert** [String, optional]: A CA certificate that can authenticate the NSX-T server certificate. **Required** if the NSX-T Manager has a self-signed SSL certificate. Must be in PEM format. + - **default_vif_type** [String, optional]: Supported Types: `PARENT`. Default VIF type attached to logical port. Available in NSX-T v2.0+. + - **auth_certificate** [String, optional]: Certificate used for certificate-based authentication. Certificate-based authentication takes precedence over username/password if both are specified. Available in v51+. + - **auth_private_key** [String, optional]: Private key file used for certificate-based authentication. Available in v51+. + - **remote_auth** [Boolean, optional]: Enables remote authentication for NSX-T via vIDM. Available in v52.1.5+ and v53.0.1+ + - **allow_overwrite** [Boolean, optional]: When enabled, the CPI sets the `X-Allow-Overwrite` header to 'true' when making NSX-T Management API requests, which allows the Management API to mutate Policy API objects. Default: `true` for backwards compatibility. Available in v91+. !!! note If the NSX-V or NSX-T Manager has a self-signed certificate, the certificate must be set in the `ca_cert` property. @@ -437,6 +442,7 @@ cloud_provider: ``` --- + ## Example Cloud Config {: #cloud-config } ```yaml @@ -494,28 +500,29 @@ compilation: ``` --- + ## Notes {: #notes } -* Assigned VM names (e.g. `vm-8dg349-s7cn74-...`) should not be manually changed since the CPI uses them to find created VMs. You can use [`bosh vms --details`](sysadmin-commands.md#health) to find which VM is assigned which job. VMs are also tagged with their assigned job, index and deployment. +- Assigned VM names (e.g. `vm-8dg349-s7cn74-...`) should not be manually changed since the CPI uses them to find created VMs. You can use [`bosh vms --details`](sysadmin-commands.md#health) to find which VM is assigned which job. VMs are also tagged with their assigned job, index and deployment. -* Storage DRS and vMotion can be used with vSphere CPI version v18 and above. For additional details see [Storage DRS and vMotion Support](vsphere-vmotion-support.md). +- Storage DRS and vMotion can be used with vSphere CPI version v18 and above. For additional details see [Storage DRS and vMotion Support](vsphere-vmotion-support.md). -* `allow_mixed_datastores` configuration has been deprecated in favor of setting same datastore pattern for `datastore_pattern` and `persistent_datastore_pattern` keys. +- `allow_mixed_datastores` configuration has been deprecated in favor of setting same datastore pattern for `datastore_pattern` and `persistent_datastore_pattern` keys. -* The vSphere CPI requires access to port 80/443 for all the ESXi hosts in your +- The vSphere CPI requires access to port 80/443 for all the ESXi hosts in your vSphere resource pool(s). In order to upload stemcells to vSphere, the vSphere CPI makes use of an API call that returns a URL that the CPI should make a `POST` request to in order to upload the stemcell. This URL could have a hostname that resolves to any one of the ESXi hosts that are associated with your vSphere resource pool(s). -* Setting `enable_auto_anti_affinity_drs_rules` to true may cause `bosh deploy` to fail after the initial deployment if there are more VMs than hosts. A workaround is to set `enable_auto_anti_affinity_drs_rules` to false to perform subsequent deployments. +- Setting `enable_auto_anti_affinity_drs_rules` to true may cause `bosh deploy` to fail after the initial deployment if there are more VMs than hosts. A workaround is to set `enable_auto_anti_affinity_drs_rules` to false to perform subsequent deployments. -* Support for specifying Datastore Clusters for ephemeral and persistent disks is available with vSphere CPI version v47 and above. For additional details see [Release Notes for v47](https://github.com/cloudfoundry/bosh-vsphere-cpi-release/releases/tag/v47) +- Support for specifying Datastore Clusters for ephemeral and persistent disks is available with vSphere CPI version v47 and above. For additional details see [Release Notes for v47](https://github.com/cloudfoundry/bosh-vsphere-cpi-release/releases/tag/v47) -* Support for specifying Datastore Clusters nested under folders is available with vSphere CPI version v94 and above. For additional details see [Release Notes for v94](https://github.com/cloudfoundry/bosh-vsphere-cpi-release/releases/tag/v94) +- Support for specifying Datastore Clusters nested under folders is available with vSphere CPI version v94 and above. For additional details see [Release Notes for v94](https://github.com/cloudfoundry/bosh-vsphere-cpi-release/releases/tag/v94) -* Starting with v98, the vSphere CPI sets the SCSI controller on cloned VMs to ParaVirtual by default, which supports up to 63 disks and improves performance in many environments. In some storage configurations, however, this may increase disk write latency. Since [PR #459](https://github.com/cloudfoundry/bosh-vsphere-cpi-release/pull/459), operators can set `default_scsi_controller_type` to `lsi_logic` or `lsi_logic_sas` in the [global CPI configuration](#global) to use a different controller type. For `bosh create-env` deployments, the property must be set in both the instance group properties and the `cloud_provider` properties. +- Starting with v98, the vSphere CPI sets the SCSI controller on cloned VMs to ParaVirtual by default, which supports up to 63 disks and improves performance in many environments. In some storage configurations, however, this may increase disk write latency. Since [PR #459](https://github.com/cloudfoundry/bosh-vsphere-cpi-release/pull/459), operators can set `default_scsi_controller_type` to `lsi_logic` or `lsi_logic_sas` in the [global CPI configuration](#global) to use a different controller type. For `bosh create-env` deployments, the property must be set in both the instance group properties and the `cloud_provider` properties. ### VMs {: #vms } @@ -609,6 +616,7 @@ The current code will not work with a datacenter inside a folder. The order of precedence for policy and datastore specified for selecting ephemeral datastores is (in order they are written from top to bottom, first being most preferred): + - Storage policy is set in vm-type - Datastores in vm-type - Storage policy in Global config diff --git a/content/vsphere-esxi-host-failure.md b/content/vsphere-esxi-host-failure.md index 4d21c5221..95fd8cf91 100644 --- a/content/vsphere-esxi-host-failure.md +++ b/content/vsphere-esxi-host-failure.md @@ -1,3 +1,5 @@ +# vSphere - Recovery from an ESXi Host Failure + !!! note Do not follow this procedure if vSphere HA is enabled and bosh-vsphere-cpi is v30+; vSphere HA will automatically move all VMs from the failed host to other good hosts. @@ -10,36 +12,32 @@ the ESXi host is unavailable. The following steps will allow the Resurrector to recreate these VMs on a healthy host. 1. Manually remove the failed Host from its cluster to force removal of all VMs - - select the ESXi host from the cluster: **vCenter → Hosts and Clusters -→ _datacenter_ → _cluster_** + - select the ESXi host from the cluster: **vCenter → Hosts and Clusters → _datacenter_ → _cluster_** - right-click the failed ESXi host - select **Remove from Inventory** -2. Re-upload all stemcells currently in use by the director +1. Re-upload all stemcells currently in use by the director - check stemcells with `bosh stemcells` - ``` - +------------------------------------------+---------------+---------+-----------------------------------------+ | Name | OS | Version | CID | - +------------------------------------------+---------------+---------+-----------------------------------------+ + |------------------------------------------|---------------|---------|-----------------------------------------| | bosh-vsphere-esxi-hvm-centos-7-go_agent | centos-7 | 3184.1 | sc-bc3d762c-71a1-4e76-ae6d-7d2d4366821b | | bosh-vsphere-esxi-ubuntu-xenial-go_agent | ubuntu-xenial | 456.3 | sc-46509b02-a164-4306-89de-99abdaffe8a8 | | bosh-vsphere-esxi-ubuntu-xenial-go_agent | ubuntu-xenial | 456.112 | sc-86d76a55-5bcb-4c12-9fa7-460edd8f94cf | | bosh-vsphere-esxi-ubuntu-xenial-go_agent | ubuntu-xenial | 621.74* | sc-97e9ba2d-6ae0-41d1-beea-082b6635e7cb | - +------------------------------------------+---------------+---------+-----------------------------------------+ - ``` + - re-upload the in-use stemcells (the ones with asterisks ('*') next to their version) with the `--fix` flag, e.g.: - ``` + ```shell bosh upload stemcell https://bosh.io/d/stemcells/bosh-vsphere-esxi-ubuntu-xenial-go_agent?v=621.74 --fix ``` -6. Wait for the resurrector to recreate the VMs. Alternatively, force a recreate using `bosh cck` + +1. Wait for the resurrector to recreate the VMs. Alternatively, force a recreate using `bosh cck` and choose the `Recreate` option for each missing VM -9. After the ESXi host has been recovered and added back to the cluster, +1. After the ESXi host has been recovered and added back to the cluster, preferably while it's in maintenance mode, delete stemcells and the powered-off [stale] VMs: - * **vCenter → Hosts and Clusters - → _datacenter_ → _cluster_** - * select the recovered ESXi host - * **Related Objects → Virtual Machines** - * delete stale VMs, i.e. VMs whose names match the pattern _vm-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx_ - * delete stale stemcells, i.e. stemcells whose names match the pattern _sc-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx_ - * VMs and stemcells can be deleted by right-clicking on them, selecting **All vCenter Actions → Delete from Disk** + - **vCenter → Hosts and Clusters → _datacenter_ → _cluster_** + - select the recovered ESXi host + - **Related Objects → Virtual Machines** + - delete stale VMs, i.e. VMs whose names match the pattern _vm-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx_ + - delete stale stemcells, i.e. stemcells whose names match the pattern _sc-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx_ + - VMs and stemcells can be deleted by right-clicking on them, selecting **All vCenter Actions → Delete from Disk** diff --git a/content/vsphere-ha.md b/content/vsphere-ha.md index 5ffa578bd..8c1904d30 100644 --- a/content/vsphere-ha.md +++ b/content/vsphere-ha.md @@ -1,21 +1,23 @@ +# vSphere High Availability + vSphere High Availability (HA) is a VMware product that detects ESXi host failure, for example host power off or network partition, and automatically restarts virtual machines on other hosts in the cluster. It can interoperate effectively with the [BOSH Resurrector](resurrector.md), which recreates VMs if the Director loses contact with a VM's BOSH Agent. !!! note This feature is available with bosh-vsphere-cpi v30+. -### vCenter Configuration +## vCenter Configuration Configure vSphere HA as follows: -* Check **Cluster* → Manage → Settings → vSphere HA → +- Check **Cluster* → Manage → Settings → vSphere HA → Edit... → Turn on vSphere HA** -* Check **Host Monitoring** +- Check **Host Monitoring** -* Ensure the Response for **Failure conditions and VM response → Host Isolation** is set to **Shut down and restart VMs** +- Ensure the Response for **Failure conditions and VM response → Host Isolation** is set to **Shut down and restart VMs** -### BOSH Director Configuration +## BOSH Director Configuration Increase the timeout values of the [BOSH Health Monitor](monitoring.md#vm) on the BOSH Director to allow for smooth interoperation between BOSH and vCenter. We recommend increasing the `agent_timeout` from the default 60s to 180s in the BOSH Director's manifest to allow vCenter time to detect the failed host: diff --git a/content/vsphere-human-readable-names.md b/content/vsphere-human-readable-names.md index 3e13cb041..8e73264e1 100644 --- a/content/vsphere-human-readable-names.md +++ b/content/vsphere-human-readable-names.md @@ -1,33 +1,37 @@ +# vSphere - Human Readable Names for VMs + !!! note This feature is available with bosh-vsphere-cpi v53+. It is disabled by default. -With this feature enabled, when a new vm is created, it will be assigned to a name like `instance-group-name_deployment-name_a81a26b3a9a8` instead of a name like `vm-d6f0f537-18cd-4a1b-b0f5-ae03e8f590e8`. +With this feature enabled, when a new VM is created, it will be assigned to a name like `instance-group-name_deployment-name_a81a26b3a9a8` instead of a name like `vm-d6f0f537-18cd-4a1b-b0f5-ae03e8f590e8`. ## Enable the feature + 1. Modify Global Configuration: -```yaml -- path: /instance_groups/name=bosh/properties/vcenter? - type: replace - value: - address: ((vcenter_ip)) - datacenters: - - clusters: - - ((vcenter_cluster)): {} - datastore_pattern: ((vcenter_ds)) - disk_path: ((vcenter_disks)) - name: ((vcenter_dc)) - persistent_datastore_pattern: ((vcenter_ds)) - template_folder: ((vcenter_templates)) - vm_folder: ((vcenter_vms)) - password: ((vcenter_password)) - user: ((vcenter_user)) - enable_human_readable_name: true -``` -2. Deploy the Director + ```yaml + - path: /instance_groups/name=bosh/properties/vcenter? + type: replace + value: + address: ((vcenter_ip)) + datacenters: + - clusters: + - ((vcenter_cluster)): {} + datastore_pattern: ((vcenter_ds)) + disk_path: ((vcenter_disks)) + name: ((vcenter_dc)) + persistent_datastore_pattern: ((vcenter_ds)) + template_folder: ((vcenter_templates)) + vm_folder: ((vcenter_vms)) + password: ((vcenter_password)) + user: ((vcenter_user)) + enable_human_readable_name: true + ``` +2. Deploy the Director ## More about human readable names + 1. Currently vSphere CPI only support names in ASCII characters. 2. The total max length kept for instance group name and deployment name is 65. If original names are longer, names will be trimmed. diff --git a/content/vsphere-migrate-datastores.md b/content/vsphere-migrate-datastores.md index 83d6d664c..64b7c1f7f 100644 --- a/content/vsphere-migrate-datastores.md +++ b/content/vsphere-migrate-datastores.md @@ -1,3 +1,5 @@ +# vSphere - Migrate Datastores + !!! note This feature is available with bosh-vsphere-cpi v9+. @@ -5,9 +7,9 @@ This topic describes how to migrate VMs and persistent disks from one datastore 1. Attach new datastore(s) to the hosts where the VMs are running while keeping the old datastore(s) attached to the same hosts. -1. Change deployment manifest for the Director to configure vSphere CPI to reference new datastore(s). +1. Change deployment manifest for the Director to configure vSphere CPI to reference new datastore(s). - ```json + ```yaml properties: vsphere: host: 172.16.68.3 diff --git a/content/vsphere-network-partition.md b/content/vsphere-network-partition.md index c4146ce1c..ca7639e66 100644 --- a/content/vsphere-network-partition.md +++ b/content/vsphere-network-partition.md @@ -1,24 +1,21 @@ +# Recovery from a vSphere Network Partitioning Fault + !!! warning Do not follow this procedure if vSphere HA is enabled and bosh-vsphere-cpi is v30+; vSphere HA will automatically recreate VMs that were on the partitioned host. This topic describes how to recreate VMs in the event of a network partition that disrupts the following: -* the vCenter's ability to communicate with an ESXi host -* the BOSH Director's ability to communicate with the VMs on that host. +- the vCenter's ability to communicate with an ESXi host +- the BOSH Director's ability to communicate with the VMs on that host. There are two options. -1. Power down the ESXi host. Follow the instructions to -[recover from an ESXi host failure](vsphere-esxi-host-failure.md) to recover -your BOSH deployment. - -2. If you cannot power down your ESXi host, then you must shut down the VMs -running on the partitioned ESXi host: - - Determine which VMs are affected by using the `bosh vms --details`; - the output should resemble the following: +1. Power down the ESXi host. Follow the instructions to [recover from an ESXi host failure](vsphere-esxi-host-failure.md) to recover your BOSH deployment. +2. If you cannot power down your ESXi host, then you must shut down the VMs running on the partitioned ESXi host: + - Determine which VMs are affected by using the `bosh vms --details`; the output should resemble the following: - ``` + ```text +------------------------------------------------+--------------------+----+---------+-------------+-----------------------------------------+--------------------------------------+--------------+--------+ | VM | State | AZ | VM Type | IPs | CID | Agent ID | Resurrection | Ignore | +------------------------------------------------+--------------------+----+---------+-------------+-----------------------------------------+--------------------------------------+--------------+--------+ @@ -29,15 +26,15 @@ running on the partitioned ESXi host: | dummy/4 (473a2bf2-7147-41d5-805a-532f27c6f833) | unresponsive agent | z1 | default | | vm-2c520edb-9202-499f-a079-b3468633bd37 | 43ff0019-2af1-4c87-944b-76aa06f97b83 | active | false | +------------------------------------------------+--------------------+----+---------+-------------+-----------------------------------------+--------------------------------------+--------------+--------+ ``` - - Connect to the partitioned ESXi host, and using the `CID` from the - previous command find the Vmids of the VMs using the `CID` from the previous command, - e.g. - ``` + - Connect to the partitioned ESXi host, and using the `CID` from the previous command find the Vmids of the VMs using the `CID` from the previous command, e.g. + + ```shell esxcli vm process list | grep -A 1 ^vm-c2d2a8ac-7afb-4875-9cf3-d69978c9e8c3 esxcli vm process list | grep -A 1 ^vm-2c520edb-9202-499f-a079-b3468633bd37 # We see that the WorldNumbers (World IDs) are 199401 & 199751, respectively esxcli vm process kill --type=force --world-id=199401 esxcli vm process kill --type=force --world-id=199751 ``` - - Follow the instructions [Recover from an ESXi host failure](vsphere-esxi-host-failure.md). + + - Follow the instructions [Recover from an ESXi host failure](vsphere-esxi-host-failure.md). diff --git a/content/vsphere-vmotion-support.md b/content/vsphere-vmotion-support.md index 184aa9275..4f6b8568c 100644 --- a/content/vsphere-vmotion-support.md +++ b/content/vsphere-vmotion-support.md @@ -1,6 +1,9 @@ +# vSphere Storage DRS and vMotion Support + This topic describes how storage vmotion: -* is used by bosh -* or may affect bosh deployments when triggered independently of bosh + +- is used by bosh +- or may affect bosh deployments when triggered independently of bosh ## Bosh director using storage vmotion to move persistent disks across data stores @@ -8,7 +11,7 @@ When updating to desired datastore for a deployment (see [Migrating Datastores]( ## Vsphere infrastructure pro-actively moving disks across data stores -It is possible for an operator to proactively move disks across datastores without coordination with the bosh director. This may be done manually, through the vsphere api (potentially through community projects, see https://github.com/vmware-tanzu/vmotion-migration-tool-for-bosh-deployments ), or through vsphere storage DRS. This does not require VMs and bosh jobs to be stopped. +It is possible for an operator to proactively move disks across datastores without coordination with the bosh director. This may be done manually, through the vsphere api (potentially through community projects, see [Vmotion migration tool](https://github.com/vmware-tanzu/vmotion-migration-tool-for-bosh-deployments) ), or through vsphere storage DRS. This does not require VMs and bosh jobs to be stopped. !!! note Storage DRS and vMotion can be used with bosh-vsphere-cpi v18+. @@ -23,4 +26,3 @@ Later versions of the CPI are able to locate disks migrated by vSphere as long a As VMs are recreated, the CPI will move persistent disks out of VM folders so that they are not deleted with the VMs. This procedure will happen automatically when VMs are deleted (in `delete_vm` CPI call) and when disks are detached (in `detach_disk` CPI call). - diff --git a/content/vsphere.md b/content/vsphere.md index acb5cea6d..dd95ef16f 100644 --- a/content/vsphere.md +++ b/content/vsphere.md @@ -1,31 +1,27 @@ ---- -title: VMware vSphere ---- - -# vSphere +# VMware vSphere The `vsphere` CPI can be used with [VMware vSphere](https://www.vmware.com/products/vsphere.html). - * Release: [cloudfoundry/bosh-vsphere-cpi-release](https://github.com/cloudfoundry/bosh-vsphere-cpi-release) - * Issues: [GitHub Issues](https://github.com/cloudfoundry/bosh-vsphere-cpi-release/issues) - * Slack: [cloudfoundry#bosh](https://cloudfoundry.slack.com/messages/bosh) +- Release: [cloudfoundry/bosh-vsphere-cpi-release](https://github.com/cloudfoundry/bosh-vsphere-cpi-release) +- Issues: [GitHub Issues](https://github.com/cloudfoundry/bosh-vsphere-cpi-release/issues) +- Slack: [cloudfoundry#bosh](https://cloudfoundry.slack.com/messages/bosh) ## Requirements An environment running one of the following supported releases: - * [vSphere 6.5](https://docs.vmware.com/en/VMware-vSphere/6.5/rn/vsphere-esxi-vcenter-server-65-release-notes.html) - * [vSphere 6.7](https://docs.vmware.com/en/VMware-vSphere/6.7/rn/vsphere-esxi-vcenter-server-67-release-notes.html) - * [vSphere 7.0](https://docs.vmware.com/en/VMware-vSphere/7.0/rn/vsphere-esxi-vcenter-server-70-release-notes.html) - * [vSphere 8.0](https://docs.vmware.com/en/VMware-vSphere/8.0/rn/vmware-vsphere-80-release-notes/index.html) +- [vSphere 6.5](https://docs.vmware.com/en/VMware-vSphere/6.5/rn/vsphere-esxi-vcenter-server-65-release-notes.html) +- [vSphere 6.7](https://docs.vmware.com/en/VMware-vSphere/6.7/rn/vsphere-esxi-vcenter-server-67-release-notes.html) +- [vSphere 7.0](https://docs.vmware.com/en/VMware-vSphere/7.0/rn/vsphere-esxi-vcenter-server-70-release-notes.html) +- [vSphere 8.0](https://docs.vmware.com/en/VMware-vSphere/8.0/rn/vmware-vsphere-80-release-notes/index.html) NSX Support: - * [NSX-V](https://docs.vmware.com/en/VMware-NSX-for-vSphere/index.html) (not actively tested) - * [NSX-T 2.5](https://docs.vmware.com/en/VMware-NSX-T-Data-Center/2.5.3/rn/VMware-NSX-T-Data-Center-253-Release-Notes.html) - * [NSX-T 3.0](https://docs.vmware.com/en/VMware-NSX-T-Data-Center/3.0/rn/VMware-NSX-T-Data-Center-303-Release-Notes.html) - * [NSX-T 3.1](https://docs.vmware.com/en/VMware-NSX-T-Data-Center/3.1/rn/VMware-NSX-T-Data-Center-3121-Release-Notes.html) - * [NSX-T 4.0](https://docs.vmware.com/en/VMware-NSX/4.0/rn/vmware-nsx-4001-release-notes/index.html) +- [NSX-V](https://docs.vmware.com/en/VMware-NSX-for-vSphere/index.html) (not actively tested) +- [NSX-T 2.5](https://docs.vmware.com/en/VMware-NSX-T-Data-Center/2.5.3/rn/VMware-NSX-T-Data-Center-253-Release-Notes.html) +- [NSX-T 3.0](https://docs.vmware.com/en/VMware-NSX-T-Data-Center/3.0/rn/VMware-NSX-T-Data-Center-303-Release-Notes.html) +- [NSX-T 3.1](https://docs.vmware.com/en/VMware-NSX-T-Data-Center/3.1/rn/VMware-NSX-T-Data-Center-3121-Release-Notes.html) +- [NSX-T 4.0](https://docs.vmware.com/en/VMware-NSX/4.0/rn/vmware-nsx-4001-release-notes/index.html) ## Concepts diff --git a/content/warden-cpi.md b/content/warden-cpi.md index b113a1adc..31ab14768 100644 --- a/content/warden-cpi.md +++ b/content/warden-cpi.md @@ -1,3 +1,5 @@ +# Warden/Garden CPI Cloud Properties + !!! note Updated for bosh-warden-cpi v28+. @@ -15,6 +17,7 @@ azs: ``` --- + ## Networks {: #networks } Currently the CPI does not support any cloud properties for networks. @@ -45,14 +48,15 @@ networks: The CPI does not support vip networks. --- + ## VM Types / VM Extensions {: #resource-pools } Schema for `cloud_properties` section: -* **ports** [Array, optional]: Allows to define port mapping between host and associated containers. Available in v30+. - * **host** [String, required]: Port or range of ports. Example: `80`. - * **container** [String, optional]: Port or range of ports. Defaults to `host` defined port or range. Example: `80`. - * **protocol** [String, optional]: Connection protocol. Defaults to `tcp`. Example: `udp`. +- **ports** [Array, optional]: Allows to define port mapping between host and associated containers. Available in v30+. + - **host** [String, required]: Port or range of ports. Example: `80`. + - **container** [String, optional]: Port or range of ports. Defaults to `host` defined port or range. Example: `80`. + - **protocol** [String, optional]: Connection protocol. Defaults to `tcp`. Example: `udp`. We may add simple load balancing via iptables for testing if ports is forwarded to multiple containers. @@ -76,6 +80,7 @@ vm_extensions: ``` --- + ## Disk Types {: #disk-pools } Currently the CPI does not support any cloud properties for disks. @@ -89,6 +94,7 @@ disk_pools: ``` --- + ## Global Configuration {: #global } The CPI uses containers to represent VMs and loopback devices to represent disks. Since the CPI can only talk to a single Garden server it can only manage resources on a single machine. @@ -118,11 +124,13 @@ properties: ``` --- + ## Example Cloud Config {: #cloud-config } See [bosh-deployment](https://github.com/cloudfoundry/bosh-deployment/blob/master/warden/cloud-config.yml). --- + ## Notes {: #notes } -* Garden server does not have a UI; however, you can use [gaol CLI](https://github.com/xoebus/gaol) to interact with it directly. +- Garden server does not have a UI; however, you can use [gaol CLI](https://github.com/xoebus/gaol) to interact with it directly. diff --git a/content/windows-sample-release.md b/content/windows-sample-release.md index 7d5000238..33cdd3b01 100644 --- a/content/windows-sample-release.md +++ b/content/windows-sample-release.md @@ -1,8 +1,11 @@ +# Sample Windows Release + This is a sample BOSH release than can be deployed using a Windows stemcell. It has a single job called `say-hello` that repeatedly prints out a message. After creating a deployment with this release and the `say-hello` job you can access the job's standard out with the `bosh log` command (see documentation on [logs](job-logs.md) for more information). --- + ## Release Structure {: #release-structure } ```shell @@ -26,6 +29,7 @@ packages/ ``` --- + ### `spec` {: #say-hello-spec } The `spec` file specifies the job name and description. It also contains the templates to render, which may depend on zero or more packages. See the documentation on [job spec files](jobs.md#spec) for more information. @@ -43,6 +47,7 @@ packages: [] ``` --- + ### `monit` {: #say-hello-monit } The `monit` file includes zero or more processes to run. Each process specifies an executable as well as any arguments and environment variables. See the documentation on [monit files](jobs.md#monit) for more information. Note, however, that Windows monit files are JSON config files for [Windows service wrapper](https://github.com/kohsuke/winsw), not config files for the monit Unix utility. @@ -63,6 +68,7 @@ The `monit` file includes zero or more processes to run. Each process specifies ``` --- + ### `start.ps1` {: #start-ps1 } The `start.ps1` script executed by the [service-wrapper](https://github.com/kohsuke/winsw) loops indefinitely while printing out a message: @@ -76,6 +82,7 @@ while ($true) ``` --- + ## Creating and Deploying the Sample Release {: #deploying } If you have the Director with a Windows stemcell uploaded, you can create the above described release with an empty `blobs.yml` and `final.yml`, then try deploying it: diff --git a/content/windows-stemcell-create.md b/content/windows-stemcell-create.md index 97a184f35..77108f206 100644 --- a/content/windows-stemcell-create.md +++ b/content/windows-stemcell-create.md @@ -1,11 +1,8 @@ ---- -title: Creating a Windows Stemcell for vSphere Using stembuild -owner: Windows ---- +# Creating a Windows Stemcell for vSphere Using stembuild This topic describes how to create a Windows Stemcell for vSphere. -## Overview +## Overview To create a BOSH stemcell for Windows on vSphere, do the following: @@ -20,31 +17,31 @@ To create a BOSH stemcell for Windows on vSphere, do the following: **If you already have a BOSH stemcell for Windows on vSphere, see [Monthly Stemcell Upgrades](#upgrade-stemcell)** -## Prerequisites +## Prerequisites -* A vSphere environment -* A Windows Server, version 1709, Windows Server, version 1803 or Windows Server, version 2019 ISO -* `stembuild` from a release in [stembuild](https://github.com/cloudfoundry/stembuild/releases) that corresponds to the operating system of your local host and the stemcell version that you want to build -* Microsoft [Local Group Policy Object Utility (LGPO)](https://www.microsoft.com/en-us/download/details.aspx?id=55319) downloaded to the same folder as your `stembuild` +- A vSphere environment +- A Windows Server, version 1709, Windows Server, version 1803 or Windows Server, version 2019 ISO +- `stembuild` from a release in [stembuild](https://github.com/cloudfoundry/stembuild/releases) that corresponds to the operating system of your local host and the stemcell version that you want to build +- Microsoft [Local Group Policy Object Utility (LGPO)](https://www.microsoft.com/en-us/download/details.aspx?id=55319) downloaded to the same folder as your `stembuild` -##Step 1: Create a Base VM for the BOSH Stemcell +## Step 1: Create a Base VM for the BOSH Stemcell This section describes how to create, configure, and verify a base VM for Windows from a volume-licensed ISO. -###Upload the ISO +### Upload the ISO To upload the Windows Server ISO to vSphere, do the following: 1. Log in to the vSphere Web Client. -

Note: The instructions in this topic are based on vSphere 6.0.

+ - Note: The instructions in this topic are based on vSphere 6.0. 1. Click **Storage** and select a datastore. 1. Select or create a folder where you want to upload the Windows Server ISO. 1. Click **Upload a File** and select the Windows Server ISO. You can use the `scp` utility instead of the vSphere Web Client to copy the file directly to the datastore server. -###Create and Customize a Base VM +### Create and Customize a Base VM To create and customize a base VM, do the following: @@ -52,29 +49,29 @@ To create and customize a base VM, do the following: 1. Right-click an object and select **New Virtual Machine** > **New Virtual Machine**. 1. On the **Select a creation type** page, select **Create a new virtual machine** and click **Next**. 1. On the **Select a name and folder** page, do the following: - 1. Enter a name for the VM. - 1. Select a location for the VM. - 1. Click **Next**. + 1. Enter a name for the VM. + 1. Select a location for the VM. + 1. Click **Next**. 1. On the **Select a compute resource** page, select a compute resource to run the VM and click **Next**. 1. On the **Select storage** page, do the following: - 1. Select a **VM Storage Policy**. - 1. Select the destination datastore for the VM configuration files and virtual disks. - 1. Click **Next**. + 1. Select a **VM Storage Policy**. + 1. Select the destination datastore for the VM configuration files and virtual disks. + 1. Click **Next**. 1. On the **Select compatibility** page, for the **Compatible with** configuration setting, select **ESXi 6.0 and later** and click **Next**. 1. On the **Select a guest OS** page, do the following step: - 1. For **Guest OS Family**, select **Windows**. - 1. For **Guest OS Version**, select **Microsoft Windows Server 2019**. If **Microsoft Windows Server 2019** is not + 1. For **Guest OS Family**, select **Windows**. + 1. For **Guest OS Version**, select **Microsoft Windows Server 2019**. If **Microsoft Windows Server 2019** is not available, select **Microsoft Windows Server 2016**. - 1. Click **Next**. + 1. Click **Next**. 1. On the **Customize hardware** page, configure the VM hardware using the information below and click **Next**. - 1. For **New Hard disk**, specify 30 GB or greater. - 1. For **New CD\DVD Drive**, do the following: - 1. Select **Datastore ISO File**. - 1. Select the Windows Server ISO file you uploaded to your datastore and click **OK**. - 1. Enable the **Connect At Power On** checkbox. + 1. For **New Hard disk**, specify 30 GB or greater. + 1. For **New CD\DVD Drive**, do the following: + 1. Select **Datastore ISO File**. + 1. Select the Windows Server ISO file you uploaded to your datastore and click **OK**. + 1. Enable the **Connect At Power On** checkbox. 1. Review the configuration settings on the **Ready to complete** page and click **Finish**. -###Install Windows Server +### Install Windows Server To install Windows Server on the base VM, do the following: @@ -83,22 +80,24 @@ To install Windows Server on the base VM, do the following: 1. Select **Custom installation**. 1. Complete the installation process and enter a password for the Administrator user. -###Verify OS +### Verify OS To verify that you are using the correct OS version, run the following PowerShell command on the base VM: -
[System.Environment]::OSVersion.Version
+```powershell +[System.Environment]::OSVersion.Version +``` The output should display the following: -
-[System.Environment]::OSVersion.Version
+```powershell
+[System.Environment]::OSVersion.Version
 Major    Minor    Build    Revision
 ----     ----     -----    --------
 10        0       17763    0
-
+``` -###Install VMware Tools +### Install VMware Tools To install VMware Tools on the base VM, do the following: @@ -106,7 +105,7 @@ To install VMware Tools on the base VM, do the following: 1. Navigate to the `D:` drive and run `setup64.exe`. 1. Restart the VM to complete the installation. -##Step 2: Install Windows Updates +## Step 2: Install Windows Updates Install Windows updates on the base VM using your preferred procedure. For example, you can install Windows updates by following the steps below. This procedure requires internet access. @@ -118,7 +117,7 @@ For example, you can install Windows updates by following the steps below. This You may need to restart the base VM while installing the updates. -##Step 3: Clone the Base VM +## Step 3: Clone the Base VM To clone the base VM, do the following in the vSphere Web Client: @@ -127,64 +126,64 @@ To clone the base VM, do the following in the vSphere Web Client: 1. Select **Clone** > **Clone to Virtual Machine**. This clone is your target VM. 1. Save the base VM. You will run Windows updates on this VM for future stemcells. -##Step 4: Construct the BOSH Stemcell +## Step 4: Construct the BOSH Stemcell !!! note - The target VM must be routable from your local host. Before running the construct command, ensure you are logged out of the target VM.

+ The target VM must be routable from your local host. Before running the `construct` command, ensure you are logged out of the target VM.

To construct the BOSH stemcell, run the following command from your local host: -``` +```powershell ./STEMBUILD-BINARY construct -vm-ip 'TARGET-VM-IP' -vm-username 'TARGET-USERNAME' -vm-password 'TARGET-VM-PASSWORD' -vcenter-url 'VCENTER-URL' -vcenter-username 'VCENTER-USERNAME' -vcenter-password 'VCENTER-PASSWORD' -vm-inventory-path 'INVENTORY-PATH' ``` Where: -* `STEMBUILD-BINARY` is the `stembuild` file for the version of your local host operating system and the +- `STEMBUILD-BINARY` is the `stembuild` file for the version of your local host operating system and the version of the stemcell that you want to build. For example, `stembuild-windows-2019-2`. -* `TARGET-VM-IP` is the IP address of your target VM. -* `TARGET-USERNAME` is the username of an account with Administrator privileges. -* `TARGET-VM-PASSWORD` is the password for the Administrator account. The password must be enclosed in single quotes. -* `VCENTER-URL` is the URL of your vCenter. -* `VCENTER-USERNAME` is the username of your account in vCenter. -* `VCENTER-PASSWORD` is your password. The password must be enclosed in single quotes. -* `INVENTORY-PATH` is the vCenter inventory path to the target VM. +- `TARGET-VM-IP` is the IP address of your target VM. +- `TARGET-USERNAME` is the username of an account with Administrator privileges. +- `TARGET-VM-PASSWORD` is the password for the Administrator account. The password must be enclosed in single quotes. +- `VCENTER-URL` is the URL of your vCenter. +- `VCENTER-USERNAME` is the username of your account in vCenter. +- `VCENTER-PASSWORD` is your password. The password must be enclosed in single quotes. +- `INVENTORY-PATH` is the vCenter inventory path to the target VM. !!! warning This operation may take up to an hour to complete and results in a powered-off target Windows VM in your vSphere environment. - During construct execution, the WinRM connection terminates. This behavior is expected, and the construct command is still being executed. - Do not attempt to re-run the construct command.

+ During `construct` execution, the WinRM connection terminates. This behavior is expected, and the `construct` command is still being executed. + Do not attempt to re-run the `construct` command. If you want to view the status of `construct`, you can log in to the target VM and do the following: 1. Start PowerShell. 1. Run the following command: - ``` + ```powershell Get-Content -Path "C:\provision\log.log" -Wait ``` -##Step 5: Package the BOSH Stemcell +## Step 5: Package the BOSH Stemcell To package the BOSH stemcell, run the following command from your local host: -``` +```powershell ./STEMBUILD-BINARY package -vcenter-url 'VCENTER-URL' -vcenter-username 'VCENTER-USERNAME' -vcenter-password 'VCENTER-PASSWORD' -vm-inventory-path 'INVENTORY-PATH' ``` Where: -* `STEMBUILD-BINARY` is the `stembuild` file for the version of your local host operating system and the +- `STEMBUILD-BINARY` is the `stembuild` file for the version of your local host operating system and the version of the stemcell that you want to build. For example, `stembuild-windows-2019-2`. -* `VCENTER-URL` is the URL of your vCenter. -* `VCENTER-USERNAME` is the username of your account in vCenter. -* `VCENTER-PASSWORD` is your password. The password must be enclosed in single quotes. -* `INVENTORY-PATH` is the vCenter inventory path to the target VM. +- `VCENTER-URL` is the URL of your vCenter. +- `VCENTER-USERNAME` is the username of your account in vCenter. +- `VCENTER-PASSWORD` is your password. The password must be enclosed in single quotes. +- `INVENTORY-PATH` is the vCenter inventory path to the target VM. -

Note: This command creates a stemcell on your local host in the folder where you ran the command and may -take up to 30 minutes to complete.

+!!! note + This command creates a stemcell on your local host in the folder where you ran the command and may take up to 30 minutes to complete. -##Monthly Stemcell Upgrades +## >Monthly Stemcell Upgrades After Microsoft releases operating system updates, you should upgrade your BOSH stemcell. Microsoft typically releases Windows updates on the second Tuesday of each month. @@ -195,4 +194,4 @@ To upgrade your BOSH stemcell, do the following: 1. [Clone the Base VM](#clone-vm). 1. [Construct the BOSH Stemcell](#construct-stemcell). 1. [Package the BOSH Stemcell](#package-stemcell). -1. Deploy the updated stemcell with BOSH. +1. Deploy the updated stemcell with BOSH. diff --git a/content/windows.md b/content/windows.md index d73f2fd4d..7fce68df0 100644 --- a/content/windows.md +++ b/content/windows.md @@ -1,9 +1,12 @@ +# Windows + BOSH can deploy jobs on Windows VMs. There is open source tooling and documentation available to build [AWS](https://github.com/cloudfoundry-incubator/aws-light-stemcell-builder), [Azure](https://github.com/cloudfoundry-incubator/bosh-windows-stemcell-builder/blob/master/azure-light-stemcell.md), [vSphere](https://github.com/cloudfoundry-incubator/bosh-windows-stemcell-builder/blob/master/create-manual-vsphere-stemcells.md) and [Openstack](https://github.com/cloudfoundry-incubator/bosh-windows-stemcell-builder/blob/master/create-manual-openstack-stemcells.md) stemcells for Windows. In general Windows BOSH Releases work in the same way as a standard BOSH release. The main difference is that the [monit file](create-release.md#monit) for Linux Releases is structured differently on Windows. Below are specific concerns for jobs on Windows. --- + ## Releases {: #releases } The structure of a BOSH release for Windows is identical to [Linux BOSH Releases](create-release.md). This means the structure of a Windows BOSH release will be: @@ -15,6 +18,7 @@ The structure of a BOSH release for Windows is identical to [Linux BOSH Releases - additional hook scripts --- + ## Jobs {: #jobs } The structure of a BOSH job for Windows is similar to the [Standard Linux BOSH Job Lifecycle](job-lifecycle.md), only with processes monitored by [Windows Service Wrapper](https://github.com/kohsuke/winsw) instead of monit. @@ -41,6 +45,7 @@ The above monit file will execute the file `C:\var\vcap\jobs\say-hello\bin\start Also, note that Pre-Start, Post-Start, Drain, and Post-Deploy scripts (described in the [job lifecycle](job-lifecycle.md)) must be powershell scripts and end with the `.ps1` extension, i.e., `pre-start.ps1`, `post-start.ps1`, `drain.ps1`, and `post-deploy.ps1`. --- + ### Stop Scripts in Jobs {: #stop-scripts } Release job can have a stop script that will run when the job is restarted or stopped. This script allows the job to clean up and get into a state where it can be safely stopped. @@ -51,7 +56,6 @@ To use a stop script, a change to the job's `monit` and `spec` file must be made Stdout and Stderr are currently not preserved. It is recommended to use the Windows EventLog. - ### Monit Monit changes can refer to separate scripts for both stop and start actions. @@ -77,13 +81,15 @@ For instance, to use separate scripts for start and stop: The `spec` file change is similar to linux. Here is an example: -``` +```yaml --- name: simple-stop-example templates: stop.ps1: bin/stop.ps1 ``` + --- + ## Sample BOSH Windows Release Please see [the next page](windows-sample-release.md) for a sample BOSH Windows release. diff --git a/mkdocs.yml b/mkdocs.yml index 969795683..25626359c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -12,9 +12,12 @@ pages: - About: - Welcome: index.md - Project Goals: problems.md + - Design: design.md + - Understanding BOSH: understanding-bosh.md - Community: community.md - Installation: - Installing the CLI: cli-v2-install.md + - CLI env Dependencies: cli-env-deps.md - Quick Start: quick-start.md - Alibaba Cloud: init-alicloud.md - Amazon Web Services: init-aws.md