[GAPRINDASHVILI] Ensure Openstack uses its own CinderManager

app/models/manageiq/providers/openstack/builder.rb

@@ -4,6 +4,14 @@ def build_inventory(ems, target)
       case target
       when ManageIQ::Providers::Openstack::CloudManager
         cloud_manager_inventory(ems, target)
+      when ManageIQ::Providers::Openstack::StorageManager::CinderManager
+        inventory(
+          ems,
+          target,
+          ManageIQ::Providers::Openstack::Inventory::Collector::StorageManager::CinderManager,
+          ManageIQ::Providers::Openstack::Inventory::Persister::StorageManager::CinderManager,
+          [ManageIQ::Providers::Openstack::Inventory::Parser::StorageManager::CinderManager]
+        )
       when ManageIQ::Providers::Openstack::NetworkManager
         inventory(
           ems,
@@ -19,7 +27,8 @@ def build_inventory(ems, target)
           ManageIQ::Providers::Openstack::Inventory::Collector::TargetCollection,
           ManageIQ::Providers::Openstack::Inventory::Persister::TargetCollection,
           [ManageIQ::Providers::Openstack::Inventory::Parser::CloudManager,
-           ManageIQ::Providers::Openstack::Inventory::Parser::NetworkManager]
+           ManageIQ::Providers::Openstack::Inventory::Parser::NetworkManager,
+           ManageIQ::Providers::Openstack::Inventory::Parser::StorageManager::CinderManager]
         )
       else
         # Fallback to ems refresh

app/models/manageiq/providers/openstack/cinder_manager_mixin.rb

@@ -0,0 +1,20 @@
+module ManageIQ::Providers::Openstack::CinderManagerMixin
+  extend ActiveSupport::Concern
+  include ::CinderManagerMixin
+
+  included do
+    # TODO: how about many storage managers???
+    # Should use has_many :storage_managers,
+    has_one  :cinder_manager,
+             :foreign_key => :parent_ems_id,
+             :class_name  => "ManageIQ::Providers::StorageManager::CinderManager",
+             :autosave    => true,
+             :dependent   => :destroy
+
+    delegate :cloud_volumes,
+             :cloud_volume_snapshots,
+             :cloud_volume_backups,
+             :to        => :cinder_manager,
+             :allow_nil => true
+  end
+end

app/models/manageiq/providers/openstack/cloud_manager.rb

@@ -30,7 +30,7 @@ class ManageIQ::Providers::Openstack::CloudManager < ManageIQ::Providers::CloudM
            :class_name  => "ManageIQ::Providers::StorageManager",
            :autosave    => true
 
-  include CinderManagerMixin
+  include ManageIQ::Providers::Openstack::CinderManagerMixin
   include SwiftManagerMixin
   include ManageIQ::Providers::Openstack::ManagerMixin
   include ManageIQ::Providers::Openstack::IdentitySyncMixin
@@ -77,7 +77,7 @@ def ensure_network_manager
 
   def ensure_cinder_manager
     return false if cinder_manager
-    build_cinder_manager(:type => 'ManageIQ::Providers::StorageManager::CinderManager')
+    build_cinder_manager(:type => 'ManageIQ::Providers::Openstack::StorageManager::CinderManager')
     true
   end
 

app/models/manageiq/providers/openstack/inventory/collector.rb

@@ -4,6 +4,7 @@ class ManageIQ::Providers::Openstack::Inventory::Collector < ManagerRefresh::Inv
 
   require_nested :CloudManager
   require_nested :NetworkManager
+  require_nested :CinderManager
   require_nested :TargetCollection
 
   attr_reader :availability_zones
@@ -25,6 +26,9 @@ class ManageIQ::Providers::Openstack::Inventory::Collector < ManagerRefresh::Inv
   attr_reader :security_groups
   attr_reader :volume_templates
   attr_reader :volume_snapshot_templates
+  attr_reader :cloud_volumes
+  attr_reader :cloud_volume_snapshots
+  attr_reader :cloud_volume_backups
 
   def initialize(_manager, _target)
     super
@@ -55,6 +59,10 @@ def initialize_inventory_sources
     @network_ports             = []
     @network_routers           = []
     @security_groups           = []
+    # cinder
+    @cloud_volumes             = []
+    @cloud_volume_snapshots    = []
+    @cloud_volume_backups      = []
   end
 
   def connection
@@ -66,6 +74,10 @@ def compute_service
     connection
   end
 
+  def cinder_service
+    @cinder_service ||= manager.openstack_handle.cinder_service
+  end
+
   def identity_service
     @identity_service ||= manager.openstack_handle.identity_service
   end

app/models/manageiq/providers/openstack/inventory/collector/storage_manager/cinder_manager.rb

@@ -0,0 +1,18 @@
+class ManageIQ::Providers::Openstack::Inventory::Collector::StorageManager::CinderManager < ManageIQ::Providers::Openstack::Inventory::Collector
+  include ManageIQ::Providers::Openstack::Inventory::Collector::HelperMethods
+
+  def cloud_volumes
+    return @cloud_volumes if @cloud_volumes.any?
+    @cloud_volumes = cinder_service.handled_list(:volumes, {}, cinder_admin?)
+  end
+
+  def cloud_volume_snapshots
+    return @cloud_volume_snapshots if @cloud_volume_snapshots.any?
+    @cloud_volume_snapshots = cinder_service.handled_list(:list_snapshots_detailed, {:__request_body_index => "snapshots"}, cinder_admin?)
+  end
+
+  def cloud_volume_backups
+    return @cloud_volume_backups if @cloud_volume_backups.any?
+    @cloud_volume_backups = cinder_service.handled_list(:list_backups_detailed, {:__request_body_index => "backups"}, cinder_admin?)
+  end
+end

app/models/manageiq/providers/openstack/inventory/collector/target_collection.rb

@@ -15,8 +15,8 @@ def references(collection)
     target.manager_refs_by_association.try(:[], collection).try(:[], :ems_ref).try(:to_a) || []
   end
 
-  def orchestration_stack_references
-    @orchestration_stack_references ||= target.targets.select { |x| x.kind_of?(ManagerRefresh::Target) && x.association == :orchestration_stacks }
+  def targets_by_association(association)
+    target.targets.select { |x| x.kind_of?(ManagerRefresh::Target) && x.association == association }.uniq { |x| x.manager_ref[:ems_ref] }
   end
 
   def name_references(collection)
@@ -78,9 +78,9 @@ def floating_ips
   end
 
   def orchestration_stacks
-    return [] if orchestration_stack_references.blank?
+    return [] if targets_by_association(:orchestration_stacks).blank?
     return @orchestration_stacks if @orchestration_stacks.any?
-    @orchestration_stacks = orchestration_stack_references.collect do |target|
+    @orchestration_stacks = targets_by_association(:orchestration_stacks).collect do |target|
       get_orchestration_stack(target.manager_ref[:ems_ref], target.options[:tenant_id])
     end.compact
   rescue Excon::Errors::Forbidden
@@ -194,6 +194,45 @@ def vms_by_id
     @vms_by_id ||= Hash[vms.collect { |s| [s.id, s] }]
   end
 
+  def cloud_volumes
+    return [] if references(:cloud_volumes).blank?
+    return @cloud_volumes if @cloud_volumes.any?
+    @cloud_volumes = targets_by_association(:cloud_volumes).collect do |target|
+      scoped_get_volume(target.manager_ref[:ems_ref], target.options[:tenant_id])
+    end.compact
+  end
+
+  def cloud_volume_snapshots
+    return [] if references(:cloud_volume_snapshots).blank?
+    return @cloud_volume_snapshots if @cloud_volume_snapshots.any?
+    @cloud_volume_snapshots = targets_by_association(:cloud_volume_snapshots).collect do |target|
+      scoped_get_snapshot(target.manager_ref[:ems_ref], target.options[:tenant_id])
+    end.compact
+  end
+
+  def cloud_volume_backups
+    return [] if references(:cloud_volume_backups).blank?
+    return @cloud_volume_backups if @cloud_volume_backups.any?
+    @cloud_volume_backups = targets_by_association(:cloud_volume_backups).collect do |target|
+      scoped_get_backup(target.manager_ref[:ems_ref], target.options[:tenant_id])
+    end.compact
+  end
+
+  def scoped_get_volume(volume_id, tenant_id)
+    tenant = memoized_get_tenant(tenant_id)
+    safe_get { @os_handle.detect_volume_service(tenant.try(:name)).volumes.get(volume_id) }
+  end
+
+  def scoped_get_snapshot(snapshot_id, tenant_id)
+    tenant = memoized_get_tenant(tenant_id)
+    safe_get { @os_handle.detect_volume_service(tenant.try(:name)).snapshots.get(snapshot_id) }
+  end
+
+  def scoped_get_backup(backup_id, tenant_id)
+    tenant = memoized_get_tenant(tenant_id)
+    safe_get { @os_handle.detect_volume_service(tenant.try(:name)).backups.get(backup_id) }
+  end
+
   private
 
   def parse_targets!
@@ -268,7 +307,7 @@ def infer_related_vm_ems_refs_db!
       stack      = vm.orchestration_stack
       all_stacks = ([stack] + (stack.try(:ancestors) || [])).compact
 
-      all_stacks.each { |s| add_simple_target!(:orchestration_stacks, s.ems_ref, :tenant_id => s.cloud_tenant.id) }
+      all_stacks.each { |s| add_simple_target!(:orchestration_stacks, s.ems_ref, :tenant_id => s.cloud_tenant.ems_ref) }
       vm.cloud_networks.collect(&:ems_ref).compact.each { |ems_ref| add_simple_target!(:cloud_networks, ems_ref) }
       vm.floating_ips.collect(&:address).compact.each { |address| add_simple_target!(:floating_ips, address) }
       vm.network_ports.collect(&:ems_ref).compact.each do |ems_ref|
@@ -277,6 +316,9 @@ def infer_related_vm_ems_refs_db!
       vm.key_pairs.collect(&:name).compact.each do |name|
         add_simple_target!(:key_pairs, name)
       end
+      vm.cloud_volumes.collect(&:ems_ref).compact.each do |ems_ref|
+        add_simple_target!(:cloud_volumes, ems_ref, :tenant_id => vm.cloud_tenant.ems_ref)
+      end
       add_simple_target!(:images, vm.parent.ems_ref) if vm.parent
       add_simple_target!(:cloud_tenants, vm.cloud_tenant.ems_ref) if vm.cloud_tenant
     end
@@ -290,6 +332,11 @@ def infer_related_vm_ems_refs_api!
       add_simple_target!(:cloud_tenants, vm.tenant_id)
       add_simple_target!(:flavors, vm.flavor["id"])
 
+      # pull the attachments from the raw attribute to avoid Fog making an unnecessary call
+      # to inflate the volumes before we need them
+      vm.attributes.fetch('os-extended-volumes:volumes_attached', []).each do |attachment|
+        add_simple_target!(:cloud_volumes, attachment["id"], :tenant_id => vm.tenant_id)
+      end
       vm.os_interfaces.each do |iface|
         add_simple_target!(:network_ports, iface.port_id)
         add_simple_target!(:cloud_networks, iface.net_id)

app/models/manageiq/providers/openstack/inventory/parser.rb

@@ -1,17 +1,5 @@
 class ManageIQ::Providers::Openstack::Inventory::Parser < ManagerRefresh::Inventory::Parser
   require_nested :CloudManager
   require_nested :NetworkManager
-  #
-  # include ManageIQ::Providers::Openstack::ParserHelperMethods
-  #
-  # def process_inventory_collection(collection, key)
-  #   (collection || []).each do |item|
-  #     new_result = yield(item)
-  #     next if new_result.blank?
-  #
-  #     raise "InventoryCollection #{key} must be defined" unless persister.collections[key]
-  #
-  #     persister.collections[key] << persister.collections[key].new_inventory_object(new_result)
-  #   end
-  # end
+  require_nested :CinderManager
 end

app/models/manageiq/providers/openstack/inventory/parser/storage_manager/cinder_manager.rb

@@ -0,0 +1,94 @@
+class ManageIQ::Providers::Openstack::Inventory::Parser::StorageManager::CinderManager < ManageIQ::Providers::Openstack::Inventory::Parser
+  def parse
+    cloud_volumes
+    cloud_volume_snapshots
+    cloud_volume_backups
+  end
+
+  def cloud_volumes
+    collector.cloud_volumes.each do |v|
+      volume = persister.cloud_volumes.find_or_build(v.id)
+      volume.type = "ManageIQ::Providers::Openstack::CloudManager::CloudVolume"
+      volume.name = volume_name(v)
+      volume.status = v.status
+      volume.bootable = v.attributes['bootable']
+      volume.creation_time = v.created_at
+      volume.description = volume_description(v)
+      volume.volume_type = v.volume_type
+      volume.size = v.size.to_i.gigabytes
+      volume.base_snapshot = persister.cloud_volume_snapshots.lazy_find(v.snapshot_id)
+      volume.cloud_tenant = persister.cloud_tenants.lazy_find(v.tenant_id)
+      volume.availability_zone = persister.availability_zones.lazy_find(v.availability_zone || "null_az")
+
+      volume_attachments(volume, v.attachments)
+    end
+  end
+
+  def cloud_volume_snapshots
+    collector.cloud_volume_snapshots.each do |s|
+      snapshot = persister.cloud_volume_snapshots.find_or_build(s['id'])
+      snapshot.type = "ManageIQ::Providers::Openstack::CloudManager::CloudVolumeSnapshot"
+      snapshot.creation_time = s['created_at']
+      snapshot.status = s['status']
+      snapshot.size = s['size'].to_i.gigabytes
+      # Supporting both Cinder v1 and Cinder v2
+      snapshot.name = s['display_name'] || s['name']
+      snapshot.description = s['display_description'] || s['description']
+      snapshot.cloud_volume = persister.cloud_volumes.lazy_find(s['volume_id'])
+      snapshot.cloud_tenant = persister.cloud_tenants.lazy_find(s['os-extended-snapshot-attributes:project_id'])
+    end
+  end
+
+  def cloud_volume_backups
+    collector.cloud_volume_backups.each do |b|
+      backup = persister.cloud_volume_backups.find_or_build(b['id'])
+      backup.type = "ManageIQ::Providers::Openstack::CloudManager::CloudVolumeBackup"
+      backup.status = b['status']
+      backup.creation_time = b['create_at']
+      backup.size = b['size'].to_i.gigabytes
+      backup.object_count = b['object_count'].to_i
+      backup.is_incremental = b['is_incremental']
+      backup.has_dependent_backups = b['has_dependent_backups']
+      # Supporting both Cinder v1 and Cinder v2
+      backup.name = b['display_name'] || b['name']
+      backup.description = b['display_description'] || b['description']
+      backup.cloud_volume = persister.cloud_volumes.lazy_find(b['volume_id'])
+      backup.availability_zone = persister.availability_zones.lazy_find(b['availability_zone'] || "null_az")
+    end
+  end
+
+  def volume_attachments(persister_volume, attachments)
+    (attachments || []).each do |a|
+      if a['device'].blank?
+        log_header = "MIQ(#{self.class.name}.#{__method__}) Collecting data for EMS name: [#{ems.name}] id: [#{ems.id}]"
+        _log.warn("#{log_header}: Volume: #{persister_volume.ems_ref}, is missing a mountpoint, skipping the volume processing")
+        _log.warn("#{log_header}: EMS: #{ems.name}, Instance: #{a['server_id']}")
+        next
+      end
+
+      dev = File.basename(a['device'])
+
+      attachment_names = {'vda' => 'Root disk', 'vdb' => 'Ephemeral disk', 'vdc' => 'Swap disk'}
+      persister.disks.find_or_build_by(
+        :hardware    => persister.hardwares.lazy_find(a["server_id"]),
+        :device_name => attachment_names.fetch(dev, dev)
+      ).assign_attributes(
+        :location        => dev,
+        :size            => persister_volume.size,
+        :device_type     => "disk",
+        :controller_type => "openstack",
+        :backing         => persister_volume
+      )
+    end
+  end
+
+  def volume_name(volume)
+    # Cinder v1 : Cinder v2
+    volume.respond_to?(:display_name) ? volume.display_name : volume.name
+  end
+
+  def volume_description(volume)
+    # Cinder v1 : Cinder v2
+    volume.respond_to?(:display_description) ? volume.display_description : volume.description
+  end
+end

app/models/manageiq/providers/openstack/inventory/persister.rb

@@ -1,6 +1,7 @@
 class ManageIQ::Providers::Openstack::Inventory::Persister < ManagerRefresh::Inventory::Persister
   require_nested :CloudManager
   require_nested :NetworkManager
+  require_nested :CinderManager
   require_nested :TargetCollection
 
   # TODO(lsmola) figure out a way to pass collector info, probably via target, then remove the below
@@ -27,4 +28,8 @@ def cloud
   def network
     ManageIQ::Providers::Openstack::InventoryCollectionDefault::NetworkManager
   end
+
+  def storage
+    ManageIQ::Providers::Openstack::InventoryCollectionDefault::StorageManager
+  end
 end

app/models/manageiq/providers/openstack/inventory/persister/storage_manager/cinder_manager.rb

@@ -0,0 +1,28 @@
+class ManageIQ::Providers::Openstack::Inventory::Persister::StorageManager::CinderManager < ManageIQ::Providers::Openstack::Inventory::Persister
+  def initialize_inventory_collections
+    add_inventory_collections(storage,
+                              %i(
+                                cloud_volumes
+                                cloud_volume_snapshots
+                                cloud_volume_backups
+                              ),
+                              :builder_params => {:ext_management_system => manager})
+
+    add_inventory_collections(cloud,
+                              %i(
+                                vms
+                                availability_zones
+                                hardwares
+                                cloud_tenants
+                              ),
+                              :parent   => manager.parent_manager,
+                              :strategy => :local_db_cache_all)
+
+    add_inventory_collections(cloud,
+                              %i(
+                                disks
+                              ),
+                              :parent   => manager.parent_manager,
+                              :complete => false)
+  end
+end