Targeted Refresh for Cloud VMs

app/models/manageiq/providers/openstack/builder.rb

@@ -0,0 +1,54 @@
+class ManageIQ::Providers::Openstack::Builder
+  class << self
+    def build_inventory(ems, target)
+      case target
+      when ManageIQ::Providers::Openstack::CloudManager
+        cloud_manager_inventory(ems, target)
+      when ManageIQ::Providers::Openstack::NetworkManager
+        inventory(
+          ems,
+          target,
+          ManageIQ::Providers::Openstack::Inventory::Collector::NetworkManager,
+          ManageIQ::Providers::Openstack::Inventory::Persister::NetworkManager,
+          [ManageIQ::Providers::Openstack::Inventory::Parser::NetworkManager]
+        )
+      when ManagerRefresh::TargetCollection
+        inventory(
+          ems,
+          target,
+          ManageIQ::Providers::Openstack::Inventory::Collector::TargetCollection,
+          ManageIQ::Providers::Openstack::Inventory::Persister::TargetCollection,
+          [ManageIQ::Providers::Openstack::Inventory::Parser::CloudManager,
+           ManageIQ::Providers::Openstack::Inventory::Parser::NetworkManager]
+        )
+      else
+        # Fallback to ems refresh
+        cloud_manager_inventory(ems, target)
+      end
+    end
+
+    private
+
+    def cloud_manager_inventory(ems, target)
+      inventory(
+        ems,
+        target,
+        ManageIQ::Providers::Openstack::Inventory::Collector::CloudManager,
+        ManageIQ::Providers::Openstack::Inventory::Persister::CloudManager,
+        [ManageIQ::Providers::Openstack::Inventory::Parser::CloudManager]
+      )
+    end
+
+    def inventory(manager, raw_target, collector_class, persister_class, parsers_classes)
+      collector = collector_class.new(manager, raw_target)
+      # TODO(lsmola) figure out a way to pass collector info, probably via target
+      persister = persister_class.new(manager, raw_target, collector)
+
+      ::ManageIQ::Providers::Openstack::Inventory.new(
+        persister,
+        collector,
+        parsers_classes.map(&:new)
+      )
+    end
+  end
+end

app/models/manageiq/providers/openstack/cloud_manager/refresher.rb

@@ -2,29 +2,80 @@ module ManageIQ::Providers
   class Openstack::CloudManager::Refresher < ManageIQ::Providers::BaseManager::Refresher
     include ::EmsRefresh::Refreshers::EmsRefresherMixin
 
+    def collect_inventory_for_targets(ems, targets)
+      targets_with_data = targets.collect do |target|
+        target_name = target.try(:name) || target.try(:event_type)
+
+        _log.info("Filtering inventory for #{target.class} [#{target_name}] id: [#{target.id}]...")
+        if ::Settings.ems.ems_openstack.try(:refresh).try(:inventory_object_refresh)
+          inventory = ManageIQ::Providers::Openstack::Builder.build_inventory(ems, target)
+        end
+
+        _log.info("Filtering inventory...Complete")
+        [target, inventory]
+      end
+
+      targets_with_data
+    end
+
     def parse_legacy_inventory(ems)
       ManageIQ::Providers::Openstack::CloudManager::RefreshParser.ems_inv_to_hashes(ems, refresher_options)
     end
 
-    # TODO(lsmola) NetworkManager, remove this once we have a full representation of the NetworkManager.
-    # NetworkManager should refresh base on its own conditions
-    def save_inventory(ems, target, hashes)
+    def save_inventory(ems, target, inventory_collections)
       super
-      EmsRefresh.queue_refresh(ems.network_manager)
-      EmsRefresh.queue_refresh(ems.cinder_manager)
-      EmsRefresh.queue_refresh(ems.swift_manager)
+      EmsRefresh.queue_refresh(ems.network_manager) if target.kind_of?(ManageIQ::Providers::BaseManager)
+      EmsRefresh.queue_refresh(ems.cinder_manager) if target.kind_of?(ManageIQ::Providers::BaseManager)
+      EmsRefresh.queue_refresh(ems.swift_manager) if target.kind_of?(ManageIQ::Providers::BaseManager)
     end
 
-    def parse_targeted_inventory(ems, target, collector)
-      if ::Settings.ems.ems_openstack.refresh.inventory_object_refresh
-        super(ems, target, collector)
-      else
-        super(ems, target, nil)
+    def parse_targeted_inventory(ems, _target, inventory)
+      log_header = format_ems_for_logging(ems)
+      _log.debug("#{log_header} Parsing inventory...")
+      hashes, = Benchmark.realtime_block(:parse_inventory) do
+        if ::Settings.ems.ems_openstack.try(:refresh).try(:inventory_object_refresh)
+          inventory.inventory_collections
+        else
+          ManageIQ::Providers::Openstack::CloudManager::RefreshParser.ems_inv_to_hashes(ems, refresher_options)
+        end
       end
+      _log.debug("#{log_header} Parsing inventory...Complete")
+
+      hashes
+    end
+
+    def preprocess_targets
+      @targets_by_ems_id.each do |ems_id, targets|
+        if targets.any? { |t| t.kind_of?(ExtManagementSystem) }
+          ems             = @ems_by_ems_id[ems_id]
+          targets_for_log = targets.map { |t| "#{t.class} [#{t.name}] id [#{t.id}] " }
+          _log.info("Defaulting to full refresh for EMS: [#{ems.name}], id: [#{ems.id}], from targets: #{targets_for_log}") if targets.length > 1
+        end
+
+        # We want all targets of class EmsEvent to be merged into one target,
+        # so they can be refreshed together, otherwise we could be missing some
+        # crosslinks in the refreshed data
+        all_targets, sub_ems_targets = targets.partition { |x| x.kind_of?(ExtManagementSystem) }
+
+        unless sub_ems_targets.blank?
+          if ::Settings.ems.ems_openstack.try(:refresh).try(:event_targeted_refresh)
+            # We can disable targeted refresh with a setting, then we will just do full ems refresh on any event
+            ems_event_collection = ManagerRefresh::TargetCollection.new(:targets    => sub_ems_targets,
+                                                                        :manager_id => ems_id)
+            all_targets << ems_event_collection
+          else
+            all_targets << @ems_by_ems_id[ems_id]
+          end
+        end
+
+        @targets_by_ems_id[ems_id] = all_targets
+      end
+
+      super
     end
 
     def post_process_refresh_classes
-      [Vm, CloudTenant]
+      [::Vm, CloudTenant]
     end
   end
 end

app/models/manageiq/providers/openstack/inventory.rb

@@ -0,0 +1,5 @@
+class ManageIQ::Providers::Openstack::Inventory < ManagerRefresh::Inventory
+  require_nested :Collector
+  require_nested :Parser
+  require_nested :Persister
+end

app/models/manageiq/providers/openstack/inventory/collector.rb

@@ -0,0 +1,87 @@
+class ManageIQ::Providers::Openstack::Inventory::Collector < ManagerRefresh::Inventory::Collector
+  include ManageIQ::Providers::Openstack::RefreshParserCommon::HelperMethods
+  include Vmdb::Logging
+
+  require_nested :CloudManager
+  require_nested :NetworkManager
+  require_nested :TargetCollection
+
+  attr_reader :availability_zones
+  attr_reader :cloud_services
+  attr_reader :tenants
+  attr_accessor :flavors
+  attr_reader :host_aggregates
+  attr_reader :key_pairs
+  attr_reader :miq_templates
+  attr_reader :orchestration_stacks
+  attr_reader :quotas
+  attr_reader :vms
+  attr_reader :vnfs
+  attr_reader :vnfds
+  attr_reader :cloud_networks
+  attr_reader :floating_ips
+  attr_reader :network_ports
+  attr_reader :network_routers
+  attr_reader :security_groups
+
+  def initialize(_manager, _target)
+    super
+
+    initialize_inventory_sources
+  end
+
+  def initialize_inventory_sources
+    # cloud
+    @availability_zones   = []
+    @cloud_services       = []
+    @tenants              = []
+    @flavors              = []
+    @host_aggregates      = []
+    @key_pairs            = []
+    @images               = []
+    @orchestration_stacks = []
+    @quotas               = []
+    @vms                  = []
+    @vnfs                 = []
+    @vnfds                = []
+    # network
+    @cloud_networks       = []
+    @floating_ips         = []
+    @network_ports        = []
+    @network_routers      = []
+    @security_groups      = []
+  end
+
+  def connection
+    @os_handle ||= manager.openstack_handle
+    @connection ||= manager.connect
+  end
+
+  def compute_service
+    connection
+  end
+
+  def identity_service
+    @identity_service ||= manager.openstack_handle.identity_service
+  end
+
+  def image_service
+    @image_service ||= manager.openstack_handle.detect_image_service
+  end
+
+  def network_service
+    @network_service ||= manager.openstack_handle.detect_network_service
+  end
+
+  def nfv_service
+    @nfv_service ||= manager.openstack_handle.detect_nfv_service
+  end
+
+  def volume_service
+    @volume_service ||= manager.openstack_handle.detect_volume_service
+  end
+
+  def orchestration_service
+    @orchestration_service ||= manager.openstack_handle.detect_orchestration_service
+  end
+end

app/models/manageiq/providers/openstack/inventory/collector/cloud_manager.rb

@@ -1,40 +1,4 @@
-class ManageIQ::Providers::Openstack::Inventory::Collector::CloudManager < ManagerRefresh::Inventory::Collector
-  include ManageIQ::Providers::Openstack::RefreshParserCommon::HelperMethods
-  include Vmdb::Logging
-
-  def connection
-    @os_handle ||= manager.openstack_handle
-    @connection ||= manager.connect
-  end
-
-  def compute_service
-    connection
-  end
-
-  def identity_service
-    @identity_service ||= manager.openstack_handle.identity_service
-  end
-
-  def image_service
-    @image_service ||= manager.openstack_handle.detect_image_service
-  end
-
-  def network_service
-    @network_service ||= manager.openstack_handle.detect_network_service
-  end
-
-  def nfv_service
-    @nfv_service ||= manager.openstack_handle.detect_nfv_service
-  end
-
-  def volume_service
-    @volume_service ||= manager.openstack_handle.detect_volume_service
-  end
-
-  def orchestration_service
-    @orchestration_service ||= manager.openstack_handle.detect_orchestration_service
-  end
-
+class ManageIQ::Providers::Openstack::Inventory::Collector::CloudManager < ManageIQ::Providers::Openstack::Inventory::Collector
   def availability_zones_compute
     @availability_zones_compute ||= safe_list { compute_service.availability_zones.summary }
   end
@@ -48,15 +12,18 @@ def availability_zones
   end
 
   def cloud_services
-    @cloud_services ||= compute_service.handled_list(:services, {}, ::Settings.ems.ems_openstack.refresh.is_admin)
+    return @cloud_services if @cloud_services.any?
+    @cloud_services = compute_service.handled_list(:services, {}, ::Settings.ems.ems_openstack.refresh.is_admin)
   end
 
   def flavors
-    @flavors ||= if ::Settings.ems.ems_openstack.refresh.is_admin
+    return @flavors if @flavors.any?
+    flavors = if ::Settings.ems.ems_openstack.refresh.is_admin
                    connection.handled_list(:flavors, {'is_public' => 'None'}, true)
                  else
                    connection.handled_list(:flavors)
                  end
+    @flavors = flavors
   end
 
   def find_flavor(flavor_id)
@@ -91,19 +58,22 @@ def flavors_by_id
   end
 
   def host_aggregates
-    @host_aggregates ||= compute_service.aggregates.all
+    return @host_aggregates if @host_aggregates.any?
+    @host_aggregates = safe_list { compute_service.aggregates.all }
   end
 
   def images
-    @images ||= if ::Settings.ems.ems_openstack.refresh.is_admin
-                  image_service.handled_list(:images, {}, true).all
-                else
-                  image_service.handled_list(:images)
-                end
+    return @images if @images.any?
+    @images = if ::Settings.ems.ems_openstack.refresh.is_admin
+                image_service.handled_list(:images, {}, true).all
+              else
+                image_service.handled_list(:images)
+              end
   end
 
   def key_pairs
-    @key_pairs ||= compute_service.handled_list(:key_pairs, {}, ::Settings.ems.ems_openstack.refresh.is_admin)
+    return @key_pairs if @key_pairs.any?
+    @key_pairs = compute_service.handled_list(:key_pairs, {}, ::Settings.ems.ems_openstack.refresh.is_admin)
   end
 
   def quotas
@@ -114,26 +84,28 @@ def quotas
     quotas
   end
 
-  def servers
-    @servers ||= compute_service.handled_list(:servers, {}, ::Settings.ems.ems_openstack.refresh.is_admin)
+  def vms
+    return @vms if @vms.any?
+    @vms = compute_service.handled_list(:servers, {}, ::Settings.ems.ems_openstack.refresh.is_admin)
   end
 
-  def servers_by_id
-    @servers_by_id ||= Hash[servers.collect { |s| [s.id, s] }]
+  def vms_by_id
+    @vms_by_id ||= Hash[vms.collect { |s| [s.id, s] }]
   end
 
   def tenants
-    @tenants ||= if ::Settings.ems.ems_openstack.refresh.is_admin
-                   identity_service.visible_tenants.select do |t|
-                     # avoid 401 Unauth errors when checking for accessible tenants
-                     # the "services" tenant is a special tenant in openstack reserved
-                     # specifically for the various services
-                     next if t.name == "services"
-                     true
-                   end
-                 else
-                   manager.openstack_handle.accessible_tenants
+    return @tenants if @tenants.any?
+    @tenants = if ::Settings.ems.ems_openstack.refresh.is_admin
+                 identity_service.visible_tenants.select do |t|
+                   # avoid 401 Unauth errors when checking for accessible tenants
+                   # the "services" tenant is a special tenant in openstack reserved
+                   # specifically for the various services
+                   next if t.name == "services"
+                   true
                  end
+               else
+                 manager.openstack_handle.accessible_tenants
+               end
   end
 
   def vnfs
@@ -151,7 +123,8 @@ def orchestration_stacks
     # TODO(lsmola) We need a support of GET /{tenant_id}/stacks/detail in FOG, it was implemented here
     # https://review.openstack.org/#/c/35034/, but never documented in API reference, so right now we
     # can't get list of detailed stacks in one API call.
-    @orchestration_stacks ||= if ::Settings.ems.ems_openstack.refresh.heat.is_global_admin
+    return @orchestration_stacks if @orchestration_stacks.any?
+    @orchestration_stacks = if ::Settings.ems.ems_openstack.refresh.heat.is_global_admin
                                 orchestration_service.handled_list(:stacks, {:show_nested => true, :global_tenant => true}, true).collect(&:details)
                               else
                                 orchestration_service.handled_list(:stacks, :show_nested => true).collect(&:details)