Performance improvement querying Active Directory

[santosshen]

@cannatag #909
I used ldap3 to query the Active Directory

ldap_resp = decode_message_fast(response)
dict_response = self.decode_response_fast(ldap_resp)

Decode_message takes too long

Total time: 0.334575 s
File: E:\virtualenv\itdev\lib\site-packages\ldap3\strategy\sync.py
Function: _get_response at line 190

Line #      Hits         Time  Per Hit   % Time  Line Contents
==============================================================
   190                                               @func_line_time
   191                                               def _get_response(self, message_id, timeout):
   192                                                   """
   193                                                   Performs the capture of LDAP response for SyncStrategy
   194                                                   """
   195         1         10.0     10.0      0.0          ldap_responses = []
   196         1          6.0      6.0      0.0          response_complete = False
   197         2          9.0      4.5      0.0          while not response_complete:
   198         1     507951.0 507951.0     15.2              responses = self.receiving()
   199         1         10.0     10.0      0.0              if responses:
   200       902       4190.0      4.6      0.1                  for response in responses:
   201       901       5447.0      6.0      0.2                      if len(response) > 0:
   202       901      12767.0     14.2      0.4                          if self.connection.usage:
   203       901      22865.0     25.4      0.7                              self.connection._usage.update_received_message(len(response))
   204       901       4390.0      4.9      0.1                          if self.connection.fast_decoder:
   205       901     995403.0   1104.8     29.8                              ldap_resp = decode_message_fast(response)
   206       901    1763882.0   1957.7     52.7                              dict_response = self.decode_response_fast(ldap_resp)
   207                                                                   else:
   208                                                                       ldap_resp, _ = decoder.decode(response, asn1Spec=LDAP_MESSAGE_TEMPLATE)  # unprocessed unused because receiving() waits for the whole message
   209                                                                       dict_response = self.decode_response(ldap_resp)
   210       901      11779.0     13.1      0.4                          if log_enabled(EXTENDED):
   211                                                                       log(EXTENDED, 'ldap message received via <%s>:%s', self.connection, format_ldap_message(ldap_resp, '<<'))
   212       901       6325.0      7.0      0.2                          if int(ldap_resp['messageID']) == message_id:
   213       901       5810.0      6.4      0.2                              ldap_responses.append(dict_response)
   214       901       4890.0      5.4      0.1                              if dict_response['type'] not in ['searchResEntry', 'searchResRef', 'intermediateResponse']:
   215         1          4.0      4.0      0.0                                  response_complete = True
   216                                                                   elif int(ldap_resp['messageID']) == 0:  # 0 is reserved for 'Unsolicited Notification' from server as per RFC4511 (paragraph 4.4)

Connection Usage:
  Time: [elapsed:          0:00:00.757611]
    Initial start time:    2024-06-06T09:43:29.451061
    Open socket time:      2024-06-06T09:43:29.451061
    Last transmitted time: 2024-06-06T09:43:30.208672
    Last received time:    2024-06-06T09:43:30.188577
    Close socket time:     2024-06-06T09:43:30.208672
  Server:
    Servers from pool:     0
    Sockets open:          1
    Sockets closed:        1
    Sockets wrapped:       0
  Bytes:                   709869
    Transmitted:           1700
    Received:              708169
  Messages:                1817
    Transmitted:           5
    Received:              1812
  Operations:              5
    Abandon:               0
    Bind:                  1
    Add:                   0
    Compare:               0
    Delete:                0
    Extended:              0
    Modify:                0
    ModifyDn:              0
    Search:                3
    Unbind:                1
  Referrals:               
    Received:              0
    Followed:              0
    Connections:           0
  Restartable tries:       0
    Failed restarts:       0
    Successful restarts:   0

[santosshen]

dict_response = self.decode_response_fast(ldap_resp) , get config and checked_attributes too long

Total time: 0.0001033 s
File: E:\virtualenv\itdev\lib\site-packages\ldap3\operation\search.py
Function: checked_attributes_to_dict_fast at line 446

Line #      Hits         Time  Per Hit   % Time  Line Contents
==============================================================
   446                                           @func_line_time
   447                                           def checked_attributes_to_dict_fast(attribute_list, schema=None, custom_formatter=None):
   448         1         22.0     22.0      2.1      conf_case_insensitive_attributes = get_config_parameter('CASE_INSENSITIVE_ATTRIBUTE_NAMES')
   449                                           
   450         1         25.0     25.0      2.4      checked_attributes = CaseInsensitiveDict() if conf_case_insensitive_attributes else dict()
   451         4         14.0      3.5      1.4      for attribute in attribute_list:
   452         3        600.0    200.0     58.1          name = to_unicode(attribute[3][0][3], from_server=True)
   453         3        369.0    123.0     35.7          checked_attributes[name] = format_attribute_values(schema, name, decode_raw_vals_fast(attribute[3][1][3]) or [], custom_formatter)
   454         1          3.0      3.0      0.3      return checked_attributes

Total time: 2.43e-05 s
File: E:\virtualenv\itdev\lib\site-packages\ldap3\utils\conv.py
Function: to_unicode at line 35

Line #      Hits         Time  Per Hit   % Time  Line Contents
==============================================================
    35                                           @func_line_time
    36                                           def to_unicode(obj, encoding=None, from_server=False):
    37                                               """Try to convert bytes (and str in python2) to unicode.
    38                                                Return object unmodified if python3 string, else raise an exception
    39                                               """
    40         1         64.0     64.0     26.3      conf_default_client_encoding = get_config_parameter('DEFAULT_CLIENT_ENCODING')
    41         1         45.0     45.0     18.5      conf_default_server_encoding = get_config_parameter('DEFAULT_SERVER_ENCODING')
    42         1         58.0     58.0     23.9      conf_additional_server_encodings = get_config_parameter('ADDITIONAL_SERVER_ENCODINGS')
    43         1         58.0     58.0     23.9      conf_additional_client_encodings = get_config_parameter('ADDITIONAL_CLIENT_ENCODINGS')
    44         1          5.0      5.0      2.1      if isinstance(obj, NUMERIC_TYPES):
    45                                                   obj = str(obj)
    46                                           
    47         1          6.0      6.0      2.5      if isinstance(obj, (bytes, bytearray)):
    48                                                   if from_server:  # data from server

def get_config_parameter(parameter):
    if parameter == 'CASE_INSENSITIVE_ATTRIBUTE_NAMES':  # Boolean
        return _CASE_INSENSITIVE_ATTRIBUTE_NAMES
    elif parameter == 'CASE_INSENSITIVE_SCHEMA_NAMES':  # Boolean
        return _CASE_INSENSITIVE_SCHEMA_NAMES
    elif parameter == 'ABSTRACTION_OPERATIONAL_ATTRIBUTE_PREFIX':  # String
        return _ABSTRACTION_OPERATIONAL_ATTRIBUTE_PREFIX
    elif parameter == 'POOLING_LOOP_TIMEOUT':  # Integer
        return _POOLING_LOOP_TIMEOUT
    elif parameter == 'RESPONSE_SLEEPTIME':  # Integer
        return _RESPONSE_SLEEPTIME
    elif parameter == 'RESPONSE_WAITING_TIMEOUT':  # Integer
        return _RESPONSE_WAITING_TIMEOUT
    elif parameter == 'SOCKET_SIZE':  # Integer
        return _SOCKET_SIZE
    elif parameter == 'CHECK_AVAILABILITY_TIMEOUT':  # Integer
        return _CHECK_AVAILABILITY_TIMEOUT
    elif parameter == 'RESTARTABLE_SLEEPTIME':  # Integer
        return _RESTARTABLE_SLEEPTIME
    elif parameter == 'RESTARTABLE_TRIES':  # Integer
        return _RESTARTABLE_TRIES
    elif parameter == 'REUSABLE_THREADED_POOL_SIZE':  # Integer
        return _REUSABLE_THREADED_POOL_SIZE
    elif parameter == 'REUSABLE_THREADED_LIFETIME':  # Integer