diff --git a/openerp/addons/base/ir/ir_model.py b/openerp/addons/base/ir/ir_model.py index 3a76ebb3ef0f..f2e304bbe9c2 100644 --- a/openerp/addons/base/ir/ir_model.py +++ b/openerp/addons/base/ir/ir_model.py @@ -18,6 +18,8 @@ from openerp.tools.safe_eval import safe_eval as eval from openerp.tools.translate import _ +from openerp.openupgrade import openupgrade_log, openupgrade + _logger = logging.getLogger(__name__) MODULE_UNINSTALL_FLAG = '_force_unlink' @@ -123,6 +125,11 @@ def _name_search(self, cr, uid, name='', args=None, operator='ilike', context=No def _drop_table(self, cr, uid, ids, context=None): for model in self.browse(cr, uid, ids, context): + # OpenUpgrade: do not run the new table cleanup + openupgrade.message( + cr, 'Unknown', False, False, + "Not dropping the table or view of model %s", model.model) + continue model_pool = self.pool[model.model] cr.execute('select relkind from pg_class where relname=%s', (model_pool._table,)) result = cr.fetchone() @@ -388,6 +395,11 @@ def _drop_column(self, cr, uid, ids, context=None): for field in self.browse(cr, uid, ids, context): if field.name in MAGIC_COLUMNS: continue + # OpenUpgrade: do not drop columns + openupgrade.message( + cr, 'Unknown', False, False, + "Not dropping the column of field %s of model %s", field.name, field.model) + continue model = self.pool[field.model] cr.execute('SELECT relkind FROM pg_class WHERE relname=%s', (model._table,)) result = cr.fetchone() @@ -1046,6 +1058,10 @@ def unlink(self, cr, uid, ids, context=None): return super(ir_model_data,self).unlink(cr, uid, ids, context=context) def _update(self,cr, uid, model, module, values, xml_id=False, store=True, noupdate=False, mode='init', res_id=False, context=None): + #OpenUpgrade: log entry (used in csv import) + if xml_id: + openupgrade_log.log_xml_id(cr, module, xml_id) + model_obj = self.pool[model] if not context: context = {} @@ -1218,6 +1234,10 @@ def unlink_if_refcount(to_unlink): _logger.info('Deleting orphan external_ids %s', external_ids) self.unlink(cr, uid, external_ids) continue + # OpenUpgrade specific start + if not self.pool.get(field.model): + continue + # OpenUpgrade specific end if field.name in openerp.models.LOG_ACCESS_COLUMNS and self.pool[field.model]._log_access: continue if field.name == 'id': @@ -1277,10 +1297,17 @@ def _process_end(self, cr, uid, modules): for (id, name, model, res_id, module) in cr.fetchall(): if (module, name) not in self.loads: if model in self.pool: - _logger.info('Deleting %s@%s (%s.%s)', res_id, model, module, name) - if self.pool[model].exists(cr, uid, [res_id], context=context): - self.pool[model].unlink(cr, uid, [res_id], context=context) - else: + _logger.info('Deleting %s@%s', res_id, model) + try: + cr.execute('SAVEPOINT ir_model_data_delete'); + self.pool[model].unlink(cr, uid, [res_id]) + cr.execute('RELEASE SAVEPOINT ir_model_data_delete') + except Exception: + cr.execute('ROLLBACK TO SAVEPOINT ir_model_data_delete'); + _logger.warning( + 'Could not delete obsolete record with id: %d of model %s\n' + 'Please refer to the log message right above', + res_id, model) bad_imd_ids.append(id) if bad_imd_ids: self.unlink(cr, uid, bad_imd_ids, context=context) diff --git a/openerp/addons/base/ir/ir_ui_view.py b/openerp/addons/base/ir/ir_ui_view.py index 5a324fdd8e1d..19d68541e631 100644 --- a/openerp/addons/base/ir/ir_ui_view.py +++ b/openerp/addons/base/ir/ir_ui_view.py @@ -295,7 +295,16 @@ def _check_xml(self, cr, uid, ids, context=None): if view.type != 'qweb': view_doc = etree.fromstring(view_arch_utf8) # verify that all fields used are valid, etc. - self.postprocess_and_fields(cr, uid, view.model, view_doc, view.id, context=context) + try: + self.postprocess_and_fields(cr, uid, view.model, view_doc, view.id, context=context) + except Exception: + # OpenUpgrade: We ignore all view errors as they are + # caused by views introduced by models not year loaded + _logger.warn( + "Can't render view %s for model: %s. If you are " + "migrating between major versions of OpenERP, " + "this is to be expected (otherwise, do not run " + "OpenUpgrade server).", view.xml_id, view.model) # RNG-based validation is not possible anymore with 7.0 forms view_docs = [view_doc] if view_docs[0].tag == 'data': @@ -307,9 +316,23 @@ def _check_xml(self, cr, uid, ids, context=None): if parse_version(version) < parse_version('7.0') and validator and not validator.validate(view_arch): for error in validator.error_log: _logger.error(tools.ustr(error)) - return False + # OpenUpgrade: We ignore all view errors as they are + # caused by views introduced by models not year loaded + _logger.warn( + "Can't render view %s for model: %s. If you are " + "migrating between major versions of OpenERP, " + "this is to be expected (otherwise, do not run " + "OpenUpgrade server).", view.xml_id, view.model) + return True if not valid_view(view_arch): - return False + # OpenUpgrade: We ignore all view errors as they are + # caused by views introduced by models not year loaded + _logger.warn( + "Can't render view %s for model: %s. If you are " + "migrating between major versions of OpenERP, " + "this is to be expected (otherwise, do not run " + "OpenUpgrade server).", view.xml_id, view.model) + return True return True _sql_constraints = [ diff --git a/openerp/models.py b/openerp/models.py index b4b89d2d30bd..66e0d38bc9c3 100644 --- a/openerp/models.py +++ b/openerp/models.py @@ -388,9 +388,40 @@ def _field_create(self, cr, context=None): (name_id, context['module'], 'ir.model', model_id) ) - cr.execute("SELECT * FROM ir_model_fields WHERE model=%s", (self._name,)) + # OpenUpgrade edit start: In rare cases, an old module defined a field + # on a model that is not defined in another module earlier in the + # chain of inheritance. Then we need to assign the ir.model.fields' + # xmlid to this other module, otherwise the column would be dropped + # when uninstalling the first module. + # An example is res.partner#display_name defined in 7.0 by + # account_report_company, but now the field belongs to the base + # module + # Given that we arrive here in order of inheritance, we simply check + # if the field's xmlid belongs to a module already loaded, and if not, + # update the record with the correct module name. + cr.execute( + "SELECT f.*, d.module, d.id as xmlid_id " + "FROM ir_model_fields f LEFT JOIN ir_model_data d " + "ON f.id=d.res_id and d.model='ir.model.fields' WHERE f.model=%s", + (self._name,)) + # OpenUpgrade edit end cols = {} for rec in cr.dictfetchall(): + # OpenUpgrade start: + if 'module' in context and\ + rec['module'] and\ + rec['name'] in self._columns.keys() and\ + rec['module'] != context.get('module') and\ + rec['module'] not in self.pool._init_modules: + _logger.info( + 'Moving XMLID for ir.model.fields record of %s#%s ' + 'from %s to %s', + self._name, rec['name'], rec['module'], context['module']) + cr.execute( + "UPDATE ir_model_data SET module=%(module)s " + "WHERE id=%(xmlid_id)s", + dict(rec, module=context['module'])) + # OpenUpgrade end cols[rec['name']] = rec ir_model_fields_obj = self.pool.get('ir.model.fields') @@ -994,6 +1025,8 @@ def log(m): position = 0 try: + # use savepoints for openupgrade instead of transactions + cr.execute('SAVEPOINT convert_records') for res_id, xml_id, res, info in self._convert_records(cr, uid, self._extract_records(cr, uid, fields, datas, context=context, log=log), @@ -1011,8 +1044,9 @@ def log(m): if context.get('defer_parent_store_computation'): self._parent_store_compute(cr) cr.commit() + cr.execute('RELEASE SAVEPOINT convert_records') except Exception, e: - cr.rollback() + cr.execute('ROLLBACK TO SAVEPOINT convert_records') return -1, {}, 'Line %d : %s' % (position + 1, tools.ustr(e)), '' if context.get('defer_parent_store_computation'): @@ -2576,11 +2610,14 @@ def _auto_init(self, cr, context=None): self._set_default_value_on_column(cr, k, context=context) # add the NOT NULL constraint try: + # use savepoints for openupgrade instead of transactions + cr.execute('SAVEPOINT add_constraint'); cr.execute('ALTER TABLE "%s" ALTER COLUMN "%s" SET NOT NULL' % (self._table, k), log_exceptions=False) - cr.commit() + cr.execute('RELEASE SAVEPOINT add_constraint'); _schema.debug("Table '%s': column '%s': added NOT NULL constraint", self._table, k) except Exception: + cr.execute('ROLLBACK TO SAVEPOINT add_constraint'); msg = "Table '%s': unable to set a NOT NULL constraint on column '%s' !\n"\ "If you want to have it, you should update the records and execute manually:\n"\ "ALTER TABLE %s ALTER COLUMN %s SET NOT NULL" @@ -2653,11 +2690,14 @@ def _auto_init(self, cr, context=None): cr.execute('CREATE INDEX "%s_%s_index" ON "%s" ("%s")' % (self._table, k, self._table, k)) if f.required: try: - cr.commit() + #use savepoints for openupgrade instead of transactions + cr.execute('SAVEPOINT add_constraint'); cr.execute('ALTER TABLE "%s" ALTER COLUMN "%s" SET NOT NULL' % (self._table, k)) _schema.debug("Table '%s': column '%s': added a NOT NULL constraint", self._table, k) + cr.execute('RELEASE SAVEPOINT add_constraint'); except Exception: + cr.execute('ROLLBACK TO SAVEPOINT add_constraint'); msg = "WARNING: unable to set column %s of table %s not null !\n"\ "Try to re-run: openerp-server --update=module\n"\ "If it doesn't work, update records and execute manually:\n"\ @@ -2862,12 +2902,14 @@ def unify_cons_text(txt): sql_actions.sort(key=lambda x: x['order']) for sql_action in [action for action in sql_actions if action['execute']]: try: + # use savepoints for openupgrade instead of transactions + cr.execute('SAVEPOINT add_constraint2'); cr.execute(sql_action['query']) - cr.commit() + cr.execute('RELEASE SAVEPOINT add_constraint2'); _schema.debug(sql_action['msg_ok']) except: _schema.warning(sql_action['msg_err']) - cr.rollback() + cr.execute('ROLLBACK TO SAVEPOINT add_constraint2'); def _execute_sql(self, cr): diff --git a/openerp/modules/graph.py b/openerp/modules/graph.py index 9b311c666de9..d7bd8aeb77c5 100644 --- a/openerp/modules/graph.py +++ b/openerp/modules/graph.py @@ -74,12 +74,23 @@ def add_modules(self, cr, module_list, force=None): force = [] packages = [] len_graph = len(self) + + # force additional dependencies for the upgrade process if given + # in config file + forced_deps = tools.config.get_misc('openupgrade', 'force_deps', '{}') + forced_deps = tools.config.get_misc('openupgrade', + 'force_deps_' + release.version, + forced_deps) + forced_deps = tools.safe_eval.safe_eval(forced_deps) + for module in module_list: # This will raise an exception if no/unreadable descriptor file. # NOTE The call to load_information_from_description_file is already # done by db.initialize, so it is possible to not do it again here. info = openerp.modules.module.load_information_from_description_file(module) + if info and info['installable']: + info['depends'].extend(forced_deps.get(module, [])) packages.append((module, info)) # TODO directly a dict, like in get_modules_with_version else: _logger.warning('module %s: not installable, skipped', module) diff --git a/openerp/modules/loading.py b/openerp/modules/loading.py index bc81d9a06bff..38b857798f3e 100644 --- a/openerp/modules/loading.py +++ b/openerp/modules/loading.py @@ -25,11 +25,13 @@ load_openerp_module, init_module_models, adapt_version from module import runs_post_install +from openerp.openupgrade import openupgrade_loading, deferred_80 + _logger = logging.getLogger(__name__) _test_logger = logging.getLogger('openerp.tests') -def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None): +def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None, upg_registry=None): """Migrates+Updates or Installs all module nodes from ``graph`` :param graph: graph of module nodes to load :param status: deprecated parameter, unused, left to avoid changing signature in 8.0 @@ -100,6 +102,12 @@ def _load_data(cr, module_name, idref, mode, kind): if kind in ('demo', 'test'): threading.currentThread().testing = False + if status is None: + status = {} + + if skip_modules is None: + skip_modules = [] + processed_modules = [] loaded_modules = [] registry = openerp.registry(cr.dbname) @@ -108,6 +116,12 @@ def _load_data(cr, module_name, idref, mode, kind): registry.clear_manual_fields() + # suppress commits to have the upgrade of one module in just one transaction + cr.commit_org = cr.commit + cr.commit = lambda *args: None + cr.rollback_org = cr.rollback + cr.rollback = lambda *args: None + # register, instantiate and initialize models for each modules t0 = time.time() t0_sql = openerp.sql_db.sql_counter @@ -116,7 +130,7 @@ def _load_data(cr, module_name, idref, mode, kind): module_name = package.name module_id = package.id - if skip_modules and module_name in skip_modules: + if module_name in skip_modules or module_name in loaded_modules: continue migrations.migrate_module(package, 'pre') @@ -134,6 +148,14 @@ def _load_data(cr, module_name, idref, mode, kind): loaded_modules.append(package.name) if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'): registry.setup_models(cr, partial=True) + # OpenUpgrade: add this module's models to the registry + local_registry = {} + for model in models: + if not model._auto: + continue + openupgrade_loading.log_model(model, local_registry) + openupgrade_loading.compare_registries( + cr, package.name, upg_registry, local_registry) init_module_models(cr, package.name, models) idref = {} @@ -160,7 +182,14 @@ def _load_data(cr, module_name, idref, mode, kind): cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id)) modobj.invalidate_cache(cr, SUPERUSER_ID, ['demo'], [module_id]) - migrations.migrate_module(package, 'post') + # OpenUpgrade: add 'try' block for logging exceptions + # as errors in post scripts seem to be dropped + try: + migrations.migrate_module(package, 'post') + except Exception as exc: + _logger.error('Error executing post migration script for module %s: %s', + package, exc) + raise # Update translations for all installed languages modobj.update_translations(cr, SUPERUSER_ID, [module_id], None, {'overwrite': openerp.tools.config["overwrite_existing_translations"]}) @@ -199,12 +228,13 @@ def _load_data(cr, module_name, idref, mode, kind): delattr(package, kind) registry._init_modules.add(package.name) - cr.commit() + cr.commit_org() _logger.log(25, "%s modules loaded in %.2fs, %s queries", len(graph), time.time() - t0, openerp.sql_db.sql_counter - t0_sql) registry.clear_manual_fields() + cr.commit = cr.commit_org cr.commit() return loaded_modules, processed_modules @@ -223,18 +253,19 @@ def _check_module_names(cr, module_names): incorrect_names = mod_names.difference([x['name'] for x in cr.dictfetchall()]) _logger.warning('invalid module names, ignored: %s', ", ".join(incorrect_names)) -def load_marked_modules(cr, graph, states, force, progressdict, report, loaded_modules, perform_checks): +def load_marked_modules(cr, graph, states, force, progressdict, report, loaded_modules, perform_checks, upg_registry): """Loads modules marked with ``states``, adding them to ``graph`` and ``loaded_modules`` and returns a list of installed/upgraded modules.""" processed_modules = [] while True: cr.execute("SELECT name from ir_module_module WHERE state IN %s" ,(tuple(states),)) module_list = [name for (name,) in cr.fetchall() if name not in graph] + module_list = openupgrade_loading.add_module_dependencies(cr, module_list) if not module_list: break graph.add_modules(cr, module_list, force) _logger.debug('Updating graph with %d more modules', len(module_list)) - loaded, processed = load_module_graph(cr, graph, progressdict, report=report, skip_modules=loaded_modules, perform_checks=perform_checks) + loaded, processed = load_module_graph(cr, graph, progressdict, report=report, skip_modules=loaded_modules, perform_checks=perform_checks, upg_registry=upg_registry) processed_modules.extend(processed) loaded_modules.extend(loaded) if not processed: @@ -248,6 +279,7 @@ def load_modules(db, force_demo=False, status=None, update_module=False): if force_demo: force.append('demo') + upg_registry = {} cr = db.cursor() try: if not openerp.modules.db.is_initialized(cr): @@ -276,7 +308,7 @@ def load_modules(db, force_demo=False, status=None, update_module=False): # processed_modules: for cleanup step after install # loaded_modules: to avoid double loading report = registry._assertion_report - loaded_modules, processed_modules = load_module_graph(cr, graph, status, perform_checks=update_module, report=report) + loaded_modules, processed_modules = load_module_graph(cr, graph, status, perform_checks=update_module, report=report, upg_registry=upg_registry) if tools.config['load_language'] or update_module: # some base models are used below, so make sure they are set up @@ -303,7 +335,18 @@ def load_modules(db, force_demo=False, status=None, update_module=False): mods = [k for k in tools.config['update'] if tools.config['update'][k]] if mods: - ids = modobj.search(cr, SUPERUSER_ID, ['&', ('state', '=', 'installed'), ('name', 'in', mods)]) + # OpenUpgrade: in standard Odoo, '--update all' just means: + # '--update base + upward (installed) dependencies. This breaks + # the chain when new glue modules are encountered. + # E.g. purchase in 8.0 depends on stock_account and report, + # both of which are new. They may be installed, but purchase as + # an upward dependency is not selected for upgrade. + # Therefore, explicitely select all installed modules for + # upgrading in OpenUpgrade. + domain = [('state', '=', 'installed')] + if 'all' not in mods: + domain.append(('name', 'in', mods)) + ids = modobj.search(cr, SUPERUSER_ID, domain) if ids: modobj.button_upgrade(cr, SUPERUSER_ID, ids) @@ -330,11 +373,11 @@ def load_modules(db, force_demo=False, status=None, update_module=False): previously_processed = len(processed_modules) processed_modules += load_marked_modules(cr, graph, ['installed', 'to upgrade', 'to remove'], - force, status, report, loaded_modules, update_module) + force, status, report, loaded_modules, update_module, upg_registry) if update_module: processed_modules += load_marked_modules(cr, graph, ['to install'], force, status, report, - loaded_modules, update_module) + loaded_modules, update_module, upg_registry) registry.setup_models(cr) @@ -363,6 +406,10 @@ def load_modules(db, force_demo=False, status=None, update_module=False): # Cleanup orphan records registry['ir.model.data']._process_end(cr, SUPERUSER_ID, processed_modules) + # OpenUpgrade: call deferred migration steps + if update_module: + deferred_80.migrate_deferred(cr, registry) + for kind in ('init', 'demo', 'update'): tools.config[kind] = {} diff --git a/openerp/modules/migration.py b/openerp/modules/migration.py index 9776a6816ad9..351638736551 100644 --- a/openerp/modules/migration.py +++ b/openerp/modules/migration.py @@ -70,8 +70,12 @@ def migrate_module(self, pkg, stage): 'pre': '[>%s]', 'post': '[%s>]', } - - if not (hasattr(pkg, 'update') or pkg.state == 'to upgrade') or pkg.state == 'to install': + # In openupgrade, remove 'or pkg.installed_version is None' + # We want to always pass in pre and post migration files and use a new + # argument in the migrate decorator (explained in the docstring) + # to decide if we want to do something if a new module is installed + # during the migration. + if not (hasattr(pkg, 'update') or pkg.state == 'to upgrade'): return def convert_version(version): @@ -112,6 +116,11 @@ def _get_migration_files(pkg, version, stage): lst.sort() return lst + def mergedict(a, b): + a = a.copy() + a.update(b) + return a + parsed_installed_version = parse_version(pkg.installed_version or '') current_version = parse_version(convert_version(pkg.data['version'])) @@ -129,31 +138,32 @@ def _get_migration_files(pkg, version, stage): name, ext = os.path.splitext(os.path.basename(pyfile)) if ext.lower() != '.py': continue - mod = fp = fp2 = None + # OpenUpgrade edit start: + # Removed a copy of migration script to temp directory + # Replaced call to load_source with load_module so frame isn't lost and breakpoints can be set + mod = fp = None try: - fp, fname = tools.file_open(pyfile, pathinfo=True) - - if not isinstance(fp, file): - # imp.load_source need a real file object, so we create - # one from the file-like object we get from file_open - fp2 = os.tmpfile() - fp2.write(fp.read()) - fp2.seek(0) + fp, pathname = tools.file_open(pyfile, pathinfo=True) try: - mod = imp.load_source(name, fname, fp2 or fp) - _logger.info('module %(addon)s: Running migration %(version)s %(name)s' % dict(strfmt, name=mod.__name__)) - migrate = mod.migrate + mod = imp.load_module(name, fp, pathname, ('.py', 'r', imp.PY_SOURCE)) + _logger.info('module %(addon)s: Running migration %(version)s %(name)s', + mergedict({'name': mod.__name__}, strfmt)) except ImportError: - _logger.exception('module %(addon)s: Unable to load %(stage)s-migration file %(file)s' % dict(strfmt, file=pyfile)) + _logger.exception('module %(addon)s: Unable to load %(stage)s-migration file %(file)s', + mergedict({'file': pyfile}, strfmt)) raise - except AttributeError: - _logger.error('module %(addon)s: Each %(stage)s-migration file must have a "migrate(cr, installed_version)" function' % strfmt) + + _logger.info('module %(addon)s: Running migration %(version)s %(name)s', + mergedict({'name': mod.__name__}, strfmt)) + + if hasattr(mod, 'migrate'): + mod.migrate(self.cr, pkg.installed_version) else: - migrate(self.cr, pkg.installed_version) + _logger.error('module %(addon)s: Each %(stage)s-migration file must have a "migrate(cr, installed_version)" function', + strfmt) finally: if fp: fp.close() - if fp2: - fp2.close() if mod: del mod + # OpenUpgrade edit end diff --git a/openerp/tools/convert.py b/openerp/tools/convert.py index 80540a98f2de..ac3c076c7219 100644 --- a/openerp/tools/convert.py +++ b/openerp/tools/convert.py @@ -39,6 +39,8 @@ unsafe_eval = eval from safe_eval import safe_eval as eval +from openerp.openupgrade import openupgrade_log + class ParseError(Exception): def __init__(self, msg, text, filename, lineno): self.msg = msg @@ -258,6 +260,7 @@ def _test_xml_id(self, xml_id): if len(id) > 64: _logger.error('id: %s is to long (max: 64)', id) + openupgrade_log.log_xml_id(self.cr, self.module, xml_id) def _tag_delete(self, cr, rec, data_node=None, mode=None): d_model = rec.get("model") diff --git a/requirements.txt b/requirements.txt index a78b49ef51b5..50a90de3b88c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -39,3 +39,4 @@ vatnumber==1.2 vobject==0.6.6 wsgiref==0.1.2 xlwt==0.7.5 +openupgradelib>=0.1.2