# Install and configure a CEPH MON node for OCI
#
# == Parameters
#
# [*block_devices*]
# List of block devices the volume node can use as LVM backend.
class oci::volume(
  $region_name              = 'RegionOne',
  $openstack_release        = undef,
  $machine_hostname         = undef,
  $machine_ip               = undef,
  $first_master             = undef,
  $first_master_ip          = undef,
  $all_masters              = undef,
  $all_masters_ip           = undef,
  $vip_hostname             = undef,
  $vip_ipaddr               = undef,
  $vip_netmask              = undef,
  $self_signed_api_cert     = true,
  $sql_vip_ip               = undef,
  $network_ipaddr           = undef,
  $network_cidr             = undef,
  $use_ssl                  = true,

  $messaging_nodes_for_notifs = false,
  $all_rabbits                = [],
  $all_rabbits_ips            = [],

  $block_devices            = undef,
  $vgname                   = undef,
  $backup_backend           = undef,

  $pass_cinder_db           = undef,
  $pass_cinder_authtoken    = undef,
  $pass_cinder_messaging    = undef,

  # Needed for notifications:
  $pass_nova_authtoken      = undef,

  $cluster_has_osds         = false,
  $ceph_fsid                = undef,
  $ceph_admin_key           = undef,
  $ceph_openstack_key       = undef,
  $ceph_mon_initial_members = undef,
  $ceph_mon_host            = undef,
  $ceph_libvirtuuid         = undef,

  $extswift_use_external        = false,
  $extswift_auth_url            = undef,
  $extswift_region              = 'RegionOne',
  $extswift_proxy_url           = undef,
  $extswift_project_name        = undef,
  $extswift_project_domain_name = undef,
  $extswift_user_name           = undef,
  $extswift_user_domain_name    = undef,
  $extswift_password            = undef,

  $lvm_backend_name                   = 'LVM_1',
  $availability_zone                  = 'nova',
  $cinder_reserved_percentage         = 10,
  $cinder_max_over_subscription_ratio = 4.0,
  $volume_copy_bps_limit              = 1073741824,

  $disable_notifications        = false,

  $cinder_enabled_backends       = 'no-override',
  $enabled_backend_list          = undef,
  $cinder_separate_volume_groups = 'no',

  # From variables.json
  $kernel_from_backports    = false,
){

  # Generic performances tweak (will default to throughput-performance)
  # class { '::oci::tuned': }

  if $messaging_nodes_for_notifs {
    $notif_rabbit_servers = $all_rabbits
  }else{
    $notif_rabbit_servers = $all_masters
  }

  if $use_ssl {
    $proto = 'https'
    $messaging_default_port = '5671'
    $messaging_notify_port = '5671'
    $api_port = 443
  } else {
    $proto = 'http'
    $messaging_default_port = '5672'
    $messaging_notify_port = '5672'
    $api_port = 80
  }
  $messaging_default_proto = 'rabbit'
  $messaging_notify_proto = 'rabbit'

  $base_url = "${proto}://${vip_hostname}"
  $keystone_auth_uri  = "${base_url}:${api_port}/identity"
  $keystone_admin_uri = "${base_url}:${api_port}/identity"

  if $self_signed_api_cert {
    $api_endpoint_ca_file = '/etc/ssl/certs/oci-pki-oci-ca-chain.pem'
  }else{
    $api_endpoint_ca_file = ''
  }
  $oci_pki_root_ca_file = '/etc/ssl/certs/oci-pki-oci-ca-chain.pem'

  class { '::oci::puppet_oci_ca_cert':
    self_signed_api_cert => $self_signed_api_cert,
  }

  if $disable_notifications {
    $notification_driver = 'noop'
  }else{
    $notification_driver = 'messagingv2'
  }

  if $disable_notifications {
    $cinder_notif_transport_url = ''
  } else {
    $cinder_notif_transport_url = os_transport_url({
                                    'transport' => $messaging_notify_proto,
                                    'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                    'port'      => $messaging_notify_port,
                                    'username'  => 'cinder',
                                    'password'  => $pass_cinder_messaging,
                                  })
  }

  # This is needed if we're running backups over Ceph.
  if $cluster_has_osds {
    class { 'ceph':
      fsid                => $ceph_fsid,
      ensure              => 'present',
      authentication_type => 'cephx',
      mon_initial_members => $ceph_mon_initial_members,
      mon_host            => $ceph_mon_host,
    }->
    ceph::key { 'client.admin':
      secret  => $ceph_admin_key,
      cap_mon => 'allow *',
      cap_osd => 'allow *',
      cap_mds => 'allow',
    }->
    ceph::key { 'client.openstack':
      secret  => $ceph_openstack_key,
      mode    => '0644',
      cap_mon => 'profile rbd',
      cap_osd => 'profile rbd pool=cinder, profile rbd pool=nova, profile rbd pool=glance, profile rbd pool=gnocchi, profile rbd pool=cinderback',
    }
  }

  include ::cinder::client
  # Cinder main class (ie: cinder-common config)
  if $openstack_release == 'rocky'{
    class { '::cinder':
      default_transport_url => os_transport_url({
        'transport' => $messaging_default_proto,
        'hosts'     => fqdn_rotate($all_masters),
        'port'      => $messaging_default_port,
        'username'  => 'cinder',
        'password'  => $pass_cinder_messaging,
      }),
      # TODO: Fix hostname !
      database_connection   => "mysql+pymysql://cinder:${pass_cinder_db}@${sql_vip_ip}/cinderdb?charset=utf8",
      rabbit_use_ssl         => $use_ssl,
      rabbit_ha_queues       => true,
      kombu_ssl_ca_certs     => $oci_pki_root_ca_file,
      amqp_sasl_mechanisms   => 'PLAIN',
      debug                 => true,
      storage_availability_zone => $availability_zone,
    }
  }else{
    if($openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri'){
      class { '::cinder':
        default_transport_url => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'cinder',
          'password'  => $pass_cinder_messaging,
        }),
        # TODO: Fix hostname !
        database_connection   => "mysql+pymysql://cinder:${pass_cinder_db}@${sql_vip_ip}/cinderdb?charset=utf8",
        rabbit_use_ssl         => $use_ssl,
        rabbit_ha_queues       => true,
        kombu_ssl_ca_certs     => $oci_pki_root_ca_file,
        amqp_sasl_mechanisms   => 'PLAIN',
        storage_availability_zone => $availability_zone,
      }
      class { '::cinder::logging':
        debug => true,
      }
      class { 'cinder::ceilometer':
        notification_transport_url => $cinder_notif_transport_url,
        notification_driver        => $notification_driver,
      }
    }else{
      if($openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'){
        class { '::cinder':
          default_transport_url => os_transport_url({
            'transport' => $messaging_default_proto,
            'hosts'     => fqdn_rotate($all_masters),
            'port'      => $messaging_default_port,
            'username'  => 'cinder',
            'password'  => $pass_cinder_messaging,
          }),
          # TODO: Fix hostname !
          database_connection   => "mysql+pymysql://cinder:${pass_cinder_db}@${sql_vip_ip}/cinderdb?charset=utf8",
          rabbit_use_ssl         => $use_ssl,
          rabbit_ha_queues       => true,
          kombu_ssl_ca_certs     => $oci_pki_root_ca_file,
          amqp_sasl_mechanisms   => 'PLAIN',
          storage_availability_zone => $availability_zone,
          notification_transport_url => $cinder_notif_transport_url,
          notification_driver        => $notification_driver,
        }
      # Yoga and up
      }else{
        class { '::cinder::db':
          database_connection   => "mysql+pymysql://cinder:${pass_cinder_db}@${sql_vip_ip}/cinderdb?charset=utf8",
        }
        class { '::cinder':
          default_transport_url => os_transport_url({
            'transport' => $messaging_default_proto,
            'hosts'     => fqdn_rotate($all_masters),
            'port'      => $messaging_default_port,
            'username'  => 'cinder',
            'password'  => $pass_cinder_messaging,
          }),
          # TODO: Fix hostname !
          rabbit_use_ssl         => $use_ssl,
          rabbit_ha_queues       => true,
          kombu_ssl_ca_certs     => $oci_pki_root_ca_file,
          amqp_sasl_mechanisms   => 'PLAIN',
          storage_availability_zone => $availability_zone,
          notification_transport_url => $cinder_notif_transport_url,
          notification_driver        => $notification_driver,
        }
      }
      class { '::cinder::logging':
        debug => true,
      }
    }
  }


  if $extswift_use_external {
    $backup_backend_real = 'cinder.backup.drivers.swift.SwiftBackupDriver'
    $provision_cinder_backup = true
  }else{
    case $backup_backend {
      'ceph': {
        $backup_backend_real = 'cinder.backup.drivers.ceph.CephBackupDriver'
        $provision_cinder_backup = false
      }
      'swift': {
        $backup_backend_real = 'cinder.backup.drivers.swift.SwiftBackupDriver'
        $provision_cinder_backup = true
      }
      'none': {
        $backup_backend_real = ''
        $provision_cinder_backup = false
      }
    }
  }

  cinder_config {
    'DEFAULT/snapshot_clone_size':       value => '200';
    'ssl/ca_file':                       value => $api_endpoint_ca_file;
  }
  cinder_config {
    "${lvm_backend_name}/goodness_function": value => '"(capabilities.total_volumes < 200) ? 100 : 50"'
  }
  if $extswift_use_external == false {
    cinder_config {
      'DEFAULT/backup_driver':             value => $backup_backend_real;
      'DEFAULT/backup_swift_auth_url':     value => $keystone_auth_uri;
      'DEFAULT/backup_swift_ca_cert_file': value => $api_endpoint_ca_file;
    }
  }else{
    cinder_config {
      'DEFAULT/backup_ceph_user':          value => 'openstack';
      'DEFAULT/backup_ceph_pool':          value => 'cinderback';
    }
  }
  # Configure the authtoken
  class { '::cinder::keystone::authtoken':
    password             => $pass_cinder_authtoken,
    auth_url             => $keystone_admin_uri,
    www_authenticate_uri => $keystone_auth_uri,
    memcached_servers    => $memcached_servers,
    cafile               => $api_endpoint_ca_file,
    region_name          => $region_name,
  }

  # Configure the [service_user].
  if($openstack_release == 'rocky' or $openstack_release == 'stein'){
    cinder_config {
      'service_user/cafile': value => $api_endpoint_ca_file;
    }
    notify { 'No service_user config for this version of OpenStack': }
  }else{
    class { '::cinder::keystone::service_user':
      auth_url     => $keystone_auth_uri,
      password     => $pass_cinder_authtoken,
      project_name => 'services',
      cafile       => $api_endpoint_ca_file,
      region_name  => $region_name,
    }
  }

  # Configure the [nova] section.
  if($openstack_release == 'rocky'){
    cinder_config {
      'nova/cafile': value => $api_endpoint_ca_file;
    }
    notify { 'No nova section config for this version of OpenStack': }
  }else{
    class { '::cinder::nova':
      auth_url     => $keystone_auth_uri,
      password     => $pass_cinder_authtoken,
      username     => 'cinder',
      project_name => 'services',
      cafile       => $api_endpoint_ca_file,
      region_name  => $region_name,
      auth_type    => 'password',
    }
  }

  # Clear volumes on delete (for data security)
  class { '::cinder::volume':
    volume_clear        => 'zero',
    volume_clear_ionice => '-c3',
  }

  # A cinder-backup service on each volume nodes
  if $provision_cinder_backup {
    class { '::cinder::backup': }
    if $extswift_use_external {
      class { '::cinder::backup::swift':
        backup_swift_url             => $extswift_proxy_url,
        backup_swift_auth_url        => "${extswift_auth_url}",
        backup_swift_container       => 'volumebackups',
        backup_swift_object_size     => 536870912,
#        backup_swift_retry_attempts  => $::os_service_default,
#        backup_swift_retry_backoff   => $::os_service_default,
        backup_swift_user_domain     => $extswift_user_domain_name,
        backup_swift_project_domain  => $extswift_project_domain_name,
        backup_swift_project         => $extswift_project_name,
        backup_compression_algorithm => 'zlib',
      }
      cinder_config {
        'DEFAULT/backup_swift_user':         value => $extswift_user_name;
        'DEFAULT/backup_swift_key':          value => $extswift_password;
        'DEFAULT/backup_swift_auth':         value => 'single_user';
        'DEFAULT/backup_swift_auth_version': value => '3';
      }
    }else{
      cinder_config {
        'DEFAULT/backup_swift_auth': value => 'per_user';
      }
    }
  }

  # Avoids Cinder to lookup for the catalogue
  class { '::cinder::glance':
    glance_api_servers => "${base_url}/image",
  }

  # Configure the LVM backend
  if $openstack_release == 'rocky' or $openstack_release == 'stein'{
    if $cinder_enabled_backends == "no-override" {
      cinder::backend::iscsi { $lvm_backend_name:
        iscsi_ip_address   => $machine_ip,
        volume_group       => $vgname,
        iscsi_protocol     => 'iscsi',
        extra_options      => {
           "${lvm_backend_name}/reserved_percentage"           => { 'value' => $cinder_reserved_percentage },
           "${lvm_backend_name}/volume_clear_size"             => { 'value' => '0' },
           "${lvm_backend_name}/max_over_subscription_ratio"   => { 'value' => $cinder_max_over_subscription_ratio },
           "${lvm_backend_name}/volume_copy_bps_limit"         => { 'value' => $volume_copy_bps_limit }
        },
        manage_volume_type => true,
      }
    }else{
      $enabled_backend_list.each |Integer $index_backend, String $custom_backend_name| {
        $this_vg_name = "${vgname}${index_backend}"
        cinder::backend::iscsi { $custom_backend_name:
          iscsi_ip_address   => $machine_ip,
          volume_group       => $this_vg_name,
          iscsi_protocol     => 'iscsi',
          extra_options      => {
             "${custom_backend_name}/reserved_percentage"           => { 'value' => $cinder_reserved_percentage },
             "${custom_backend_name}/volume_clear_size"             => { 'value' => '0' },
             "${custom_backend_name}/max_over_subscription_ratio"   => { 'value' => $cinder_max_over_subscription_ratio },
             "${custom_backend_name}/volume_copy_bps_limit"         => { 'value' => $volume_copy_bps_limit }
          },
          manage_volume_type => true,
        }
      }
    }
  }else{
    if $cinder_enabled_backends == "no-override" {
      cinder::backend::iscsi { $lvm_backend_name:
        target_ip_address  => $machine_ip,
        volume_group       => $vgname,
        target_protocol    => 'iscsi',
        extra_options      => {
           "${lvm_backend_name}/reserved_percentage"           => { 'value' => $cinder_reserved_percentage },
           "${lvm_backend_name}/volume_clear_size"             => { 'value' => '0' },
           "${lvm_backend_name}/max_over_subscription_ratio"   => { 'value' => $cinder_max_over_subscription_ratio },
           "${lvm_backend_name}/volume_copy_bps_limit"         => { 'value' => $volume_copy_bps_limit }
        },
        manage_volume_type => true,
      }
    }else{
      $enabled_backend_list.each |Integer $index_backend, String $custom_backend_name| {
        $this_vg_name = "${vgname}${index_backend}"
        cinder::backend::iscsi { $custom_backend_name:
          target_ip_address  => $machine_ip,
          volume_group       => $this_vg_name,
          target_protocol    => 'iscsi',
          extra_options      => {
             "${custom_backend_name}/reserved_percentage"           => { 'value' => $cinder_reserved_percentage },
             "${custom_backend_name}/volume_clear_size"             => { 'value' => '0' },
             "${custom_backend_name}/max_over_subscription_ratio"   => { 'value' => $cinder_max_over_subscription_ratio },
             "${custom_backend_name}/volume_copy_bps_limit"         => { 'value' => $volume_copy_bps_limit }
          },
          manage_volume_type => true,
        }
      }
    }
  }

  # Configure Ceph backend, so that "openstack volume retype" can work
  if $cluster_has_osds {
    cinder::backend::rbd { 'CEPH_1':
      rbd_user           => 'openstack',
      rbd_pool           => 'cinder',
      rbd_secret_uuid    => $ceph_libvirtuuid,
      manage_volume_type => true,
      backend_host       => $machine_hostname,
    }
  }

  if $cinder_enabled_backends == "no-override" {
    if $cluster_has_osds {
      $lvm_backend_name_real = concat([ $lvm_backend_name] , [ 'CEPH_1' ] )
    }else{
      $lvm_backend_name_real = [ $lvm_backend_name ]
    }
    class { '::cinder::backends':
      enabled_backends => [ $lvm_backend_name_real ],
    }
  }else{
    class { '::cinder::backends':
      enabled_backends => $enabled_backend_list,
    }
  }

  # Set cgroupv1 config for volume_copy_bps_limit
  exec { 'create cinder-volume-copy cgroupv1':
    path => [ '/usr/bin', '/bin', '/usr/sbin', '/sbin' ],
    command => 'cgcreate -g blkio:cinder-volume-copy',
    unless  => 'cgget -g blkio:cinder-volume-copy 2> /dev/null | grep -q blkio.throttle',
  }

}
