Mangling source entries after reading

diff --git a/lib/ldap_migration/__init__.py b/lib/ldap_migration/__init__.py
index c5ca22a6f17074968a52f68ffa760fe31ddedfaa..56b126af45a8217fb476d6bada4428501150326a 100644 (file)
--- a/lib/ldap_migration/__init__.py
+++ b/lib/ldap_migration/__init__.py
@@ -17,6 +17,8 @@ import time
 import re
 import argparse
 
+from numbers import Number
+
 # 3rd party modules
 
 from ldap3 import Server, Connection, ALL, DSA, IP_V4_PREFERRED, SAFE_SYNC
@@ -29,7 +31,7 @@ from ldap3.core.exceptions import LDAPException
 # Own modules
 from fb_tools.colored import ColoredFormatter
 from fb_tools.errors import IoTimeoutError
-from fb_tools.common import pp, is_sequence
+from fb_tools.common import pp, is_sequence, human2mbytes
 from fb_tools.app import BaseApplication, DirectoryOptionAction
 from fb_tools.config import CfgFileOptionAction
 from fb_tools.errors import FbAppError
@@ -38,7 +40,7 @@ from .config import LDAPMigrationConfiguration
 from .idict import CaseInsensitiveDict
 from .istringset import CaseInsensitiveStringSet
 
-__version__ = '0.6.7'
+__version__ = '0.6.8'
 
 LOG = logging.getLogger(__name__)
 CFG_BASENAME = 'ldap-migration.ini'
@@ -55,6 +57,12 @@ class CommonLDAPMigrationError(FbAppError, LDAPException):
     pass
 
 
+# =============================================================================
+class ReadLDAPItemError(CommonLDAPMigrationError):
+    """Error class in cae, a LDAP item could not be read."""
+    pass
+
+
 # =============================================================================
 class NonNegativeItegerOptionAction(argparse.Action):
 
@@ -496,6 +504,99 @@ class LDAPMigrationApplication(BaseApplication):
             LOG.info("Disconnecting from source server {!r} ...".format(self.config.src_server))
             self.src_server = None
 
+    # -------------------------------------------------------------------------
+    def get_source_item(self, src_dn, tgt_dn, with_acl=False):
+        """Reading a single LDAP item."""
+
+        # ReadLDAPItemError
+        src_entry = None
+
+        sfilter = '(objectClass=*)'
+        src_attrs = [ALL_ATTRIBUTES]
+        if with_acl:
+            src_attrs = ['aci', ALL_ATTRIBUTES]
+
+        if self.verbose > 2:
+            msg = "Trying to get source LDAP item {!r} ...".format(tgt_dn)
+            LOG.debug(msg)
+
+        src_status, src_result, src_response, _ = self.source.search(
+            search_base=tgt_dn, search_scope=BASE, search_filter=sfilter,
+            attributes=src_attrs, time_limit=self.config.timeout)
+
+        if not src_status:
+            msg = "Error retrieving source LDAP item {dn!r}: {res}".format(
+                dn=tgt_dn, res=src_result)
+            raise ReadLDAPItemError(msg)
+
+        src_entry = src_response[0]
+
+        if self.verbose > 3:
+            LOG.debug("Result of searching for source DN {dn!r}:\n{res}".format(
+                dn=tgt_dn, res=pp(src_result)))
+        if self.verbose > 4:
+            LOG.debug("Response of searching for source DN {dn!r}:\n{res}".format(
+                dn=tgt_dn, res=pp(src_entry)))
+
+        entry = {
+            'attributes': {},
+            'dn': src_entry['dn'],
+            'raw_attributes': copy.copy(src_entry['raw_attributes']),
+            'raw_dn': src_entry['raw_dn'],
+            'type':src_entry['type'],
+        }
+
+        for attribute in src_entry['attributes'].keys():
+            key = self.attribute_types.get_key(attribute, strict=False)
+            if not key:
+                if self.verbose > 1:
+                    msg = (
+                        "Attribute {attr!r} of item {dn!r} not found in the list of available "
+                        "attribute types.").format(attr=attribute, dn=src_dn)
+                    LOG.warn(msg)
+                continue
+
+            src_val = src_entry['attributes'][attribute]
+            val = 0
+
+            if attribute in self.integer_attribute_types:
+
+                if is_sequence(src_val):
+                    val = []
+                    for value in src_val:
+                        if isinstance(value, Number):
+                            val.append(value)
+                            continue
+                        try:
+                            mbytes = human2mbytes(value, as_float=True)
+                            val.append(int(mbytes * 1024 * 1024))
+                        except ValueError as e:
+                            msg = "Invalid value in attribute {attr!r} of item {dn!r}: {e}"
+                            msg = msg.format(attr=attribute, dn=src_dn, e=e)
+                            raise ReadLDAPItemError(msg)
+                else:
+                    if isinstance(src_val, Number):
+                        val = src_val
+                    try:
+                        mbytes = human2mbytes(src_val, as_float=True)
+                        val = int(mbytes * 1024 * 1024)
+                    except ValueError as e:
+                        msg = "Invalid value in attribute {attr!r} of item {dn!r}: {e}"
+                        msg = msg.format(attr=attribute, dn=src_dn, e=e)
+                        raise ReadLDAPItemError(msg)
+
+                if self.verbose > 1:
+                    msg = "Migrated integer value: {old!r} => {new!r}.".format(
+                        old=src_val, new=val)
+                    LOG.warn(msg)
+
+            else:
+                val = copy.copy(src_val)
+
+            entry['attributes'][key] = val
+
+        return entry
+
     # -------------------------------------------------------------------------
     def discover_target_schema(self):
 
@@ -844,9 +945,11 @@ class LDAPMigrationApplication(BaseApplication):
         print()
         LOG.info("Migrating all entries from source to target LDAP cluster.")
 
-        self.migrate_structural_entries()
+        if not self.migrate_structural_entries():
+            return False
 
         print()
+        return True
 
     # -------------------------------------------------------------------------
     def migrate_structural_entries(self):
@@ -858,7 +961,12 @@ class LDAPMigrationApplication(BaseApplication):
         self.count_added = 0
         self.count_modified = 0
 
-        self._migrate_entries(self.struct_dns, is_root=True, with_acl=False)
+        try:
+            self._migrate_entries(self.struct_dns, is_root=True, with_acl=False)
+        except ReadLDAPItemError as e:
+            msg = "Abort migration: " + str(e)
+            LOG.error(msg)
+            return False
 
         print()
         total = self.count_unchanged + self.count_added + self.count_modified
@@ -868,6 +976,7 @@ class LDAPMigrationApplication(BaseApplication):
                     to=total, ad=self.count_added, mo=self.count_modified,
                     un=self.count_unchanged)
         LOG.info(msg)
+        return True
 
     # -------------------------------------------------------------------------
     def generate_target_entry(self, src_entry, src_dn, tgt_dn):
@@ -994,82 +1103,60 @@ class LDAPMigrationApplication(BaseApplication):
 
             src_entry = None
             tgt_entry = None
-
             sfilter = '(objectClass=*)'
-            src_attrs = [ALL_ATTRIBUTES]
-            if with_acl:
-                src_attrs = ['aci', ALL_ATTRIBUTES]
-            tgt_attrs = [ALL_ATTRIBUTES]
 
             tgt_dn = self.mangle_dn(src_dn)
+            src_entry = self.get_source_item(src_dn, tgt_dn, with_acl=with_acl)
+
+            tgt_attrs = [ALL_ATTRIBUTES]
 
-            src_status, src_result, src_response, _ = self.source.search(
+            if self.verbose > 1:
+                LOG.debug("Searching for target DN {dn!r}.".format(dn=tgt_dn))
+            tgt_status, tgt_result, tgt_response, _ = self.target.search(
                 search_base=tgt_dn, search_scope=BASE, search_filter=sfilter,
-                get_operational_attributes=True, attributes=src_attrs,
+                get_operational_attributes=with_acl, attributes=tgt_attrs,
                 time_limit=self.config.timeout)
 
-            if src_status:
-
-                src_entry = src_response[0]
-
+            target_entry = None
+            if tgt_status:
+                target_entry = tgt_response[0]
                 if self.verbose > 2:
-                    LOG.debug("Result of searching for source DN {dn!r}:\n{res}".format(
-                        dn=src_dn, res=pp(src_result)))
+                    LOG.debug("Result of searching for target DN {dn!r}:\n{res}".format(
+                        dn=tgt_dn, res=pp(tgt_result)))
                 if self.verbose > 2:
-                    LOG.debug("Response of searching for source DN {dn!r}:\n{res}".format(
-                        dn=src_dn, res=pp(src_entry)))
-
-                if self.verbose > 1:
-                    LOG.debug("Searching for target DN {dn!r}.".format(dn=tgt_dn))
-                tgt_status, tgt_result, tgt_response, _ = self.target.search(
-                    search_base=tgt_dn, search_scope=BASE, search_filter=sfilter,
-                    get_operational_attributes=with_acl, attributes=tgt_attrs,
-                    time_limit=self.config.timeout)
-
-                target_entry = None
-                if tgt_status:
-                    target_entry = tgt_response[0]
-                    if self.verbose > 2:
-                        LOG.debug("Result of searching for target DN {dn!r}:\n{res}".format(
-                            dn=tgt_dn, res=pp(tgt_result)))
-                    if self.verbose > 2:
-                        LOG.debug("Response of searching for target DN {dn!r}:\n{res}".format(
-                            dn=tgt_dn, res=pp(target_entry)))
-                    changes = self.generate_modify_data(src_entry, target_entry, src_dn, tgt_dn)
-                    if changes:
-                        if self.verbose:
-                            LOG.info("Updating target entry {!r} ...".format(tgt_dn))
-                        if self.verbose > 2:
-                            msg = "Changes on target entry {tdn!r}:\n{ch}".format(
-                                tdn=tgt_dn, ch=pp(changes))
-                            LOG.debug(msg)
-                        self.count_modified += 1
-                        if not self.simulate:
-                            self.target.modify(tgt_dn, changes)
-                        if wait:
-                            time.sleep(wait)
-
-                else:
-                    if self.verbose > 2:
-                        LOG.debug("Target DN {dn!r} not found.".format(dn=tgt_dn))
-                    (tgt_obj_classes, tgt_entry) = self.generate_target_entry(src_entry, src_dn, tgt_dn)
+                    LOG.debug("Response of searching for target DN {dn!r}:\n{res}".format(
+                        dn=tgt_dn, res=pp(target_entry)))
+                changes = self.generate_modify_data(src_entry, target_entry, src_dn, tgt_dn)
+                if changes:
                     if self.verbose:
-                        LOG.info("Creating target entry {!r} ...".format(tgt_dn))
+                        LOG.info("Updating target entry {!r} ...".format(tgt_dn))
                     if self.verbose > 2:
-                        msg = "Generated entry for target DN {dn!r}:\n"
-                        msg += "object classes: {oc}\n"
-                        msg += "entry: {en}"
-                        msg = msg.format(dn=tgt_dn, oc=tgt_obj_classes, en=tgt_entry)
+                        msg = "Changes on target entry {tdn!r}:\n{ch}".format(
+                            tdn=tgt_dn, ch=pp(changes))
                         LOG.debug(msg)
-                    self.count_added += 1
+                    self.count_modified += 1
                     if not self.simulate:
-                        self.target.add(tgt_dn, object_class=tgt_obj_classes, attributes=tgt_entry)
+                        self.target.modify(tgt_dn, changes)
                     if wait:
                         time.sleep(wait)
 
             else:
-                msg = "Did not found source entry with DN {!r} (WTF?).".format(src_dn)
-                LOG.error(msg)
+                if self.verbose > 2:
+                    LOG.debug("Target DN {dn!r} not found.".format(dn=tgt_dn))
+                (tgt_obj_classes, tgt_entry) = self.generate_target_entry(src_entry, src_dn, tgt_dn)
+                if self.verbose:
+                    LOG.info("Creating target entry {!r} ...".format(tgt_dn))
+                if self.verbose > 2:
+                    msg = "Generated entry for target DN {dn!r}:\n"
+                    msg += "object classes: {oc}\n"
+                    msg += "entry: {en}"
+                    msg = msg.format(dn=tgt_dn, oc=tgt_obj_classes, en=tgt_entry)
+                    LOG.debug(msg)
+                self.count_added += 1
+                if not self.simulate:
+                    self.target.add(tgt_dn, object_class=tgt_obj_classes, attributes=tgt_entry)
+                if wait:
+                    time.sleep(wait)
 
         for key in cur_hash['childs'].keys():
             self._migrate_entries(cur_hash['childs'][key], is_root=False, with_acl=with_acl)