Merge remote-tracking branch 'odoo-latvia/8.0' into 8.0
Sandy Carter
commit
121d09f17b
|
|
@ -20,17 +20,23 @@
|
|||
<record model="ir.ui.view" id="view_dbsource_form">
|
||||
<field name="name">base.external.dbsource.form</field>
|
||||
<field name="model">base.external.dbsource</field>
|
||||
<field name="type">form</field>
|
||||
<field name="arch" type="xml">
|
||||
<form string="External DB Source">
|
||||
<form string="External DB Source" version="7.0">
|
||||
<sheet>
|
||||
<group>
|
||||
<group>
|
||||
<field name="name"/>
|
||||
</group>
|
||||
<group>
|
||||
<field name="password" password="True"/>
|
||||
<newline/>
|
||||
<field name="connector" colspan="2"/>
|
||||
<newline/>
|
||||
<field name="conn_string" colspan="4"/>
|
||||
<newline/>
|
||||
<button name="connection_test" string="Test Connection" type="object" icon="gtk-network" colspan="4"/>
|
||||
</group>
|
||||
</group>
|
||||
<group col="1">
|
||||
<field name="connector"/>
|
||||
<field name="conn_string"/>
|
||||
<button name="connection_test" string="Test Connection" type="object" icon="gtk-network"/>
|
||||
</group>
|
||||
</sheet>
|
||||
</form>
|
||||
</field>
|
||||
</record>
|
||||
|
Before Width: | Height: | Size: 31 KiB After Width: | Height: | Size: 31 KiB |
|
Before Width: | Height: | Size: 19 KiB After Width: | Height: | Size: 19 KiB |
|
Before Width: | Height: | Size: 24 KiB After Width: | Height: | Size: 24 KiB |
|
|
@ -35,25 +35,38 @@ class import_odbc_dbtable(orm.Model):
|
|||
_columns = {
|
||||
'name': fields.char('Datasource name', required=True, size=64),
|
||||
'enabled': fields.boolean('Execution enabled'),
|
||||
'dbsource_id': fields.many2one('base.external.dbsource', 'Database source', required=True),
|
||||
'sql_source': fields.text('SQL', required=True, help='Column names must be valid "import_data" columns.'),
|
||||
'dbsource_id': fields.many2one('base.external.dbsource',
|
||||
'Database source', required=True),
|
||||
'sql_source': fields.text('SQL', required=True,
|
||||
help='Column names must be valid \
|
||||
"import_data" columns.'),
|
||||
'model_target': fields.many2one('ir.model', 'Target object'),
|
||||
'noupdate': fields.boolean('No updates', help="Only create new records; disable updates to existing records."),
|
||||
'exec_order': fields.integer('Execution order', help="Defines the order to perform the import"),
|
||||
'noupdate': fields.boolean('No updates',
|
||||
help="Only create new records;\
|
||||
disable updates to existing records."),
|
||||
'exec_order': fields.integer('Execution order',
|
||||
help="Defines the order to perform \
|
||||
the import"),
|
||||
'last_sync': fields.datetime('Last sync date',
|
||||
help="Datetime for the last succesfull sync."
|
||||
"\nLater changes on the source may not be replicated on the destination"),
|
||||
help="Datetime for the last succesfull \
|
||||
sync. \nLater changes on the source may \
|
||||
not be replicated on the destination"),
|
||||
'start_run': fields.datetime('Time started', readonly=True),
|
||||
'last_run': fields.datetime('Time ended', readonly=True),
|
||||
'last_record_count': fields.integer('Last record count', readonly=True),
|
||||
'last_record_count': fields.integer('Last record count',
|
||||
readonly=True),
|
||||
'last_error_count': fields.integer('Last error count', readonly=True),
|
||||
'last_warn_count': fields.integer('Last warning count', readonly=True),
|
||||
'last_log': fields.text('Last run log', readonly=True),
|
||||
'ignore_rel_errors': fields.boolean('Ignore relationship errors',
|
||||
help="On error try to reimport rows ignoring relationships."),
|
||||
help="On error try to reimport \
|
||||
rows ignoring relationships."),
|
||||
'raise_import_errors': fields.boolean('Raise import errors',
|
||||
help="Import errors not handled, intended for debugging purposes."
|
||||
"\nAlso forces debug messages to be written to the server log."),
|
||||
help="Import errors not \
|
||||
handled, intended for \
|
||||
debugging purposes. \nAlso \
|
||||
forces debug messages to be \
|
||||
written to the server log."),
|
||||
}
|
||||
_defaults = {
|
||||
'enabled': True,
|
||||
|
|
@ -72,21 +85,26 @@ class import_odbc_dbtable(orm.Model):
|
|||
|
||||
def append_to_log(log, level, obj_id='', msg='', rel_id=''):
|
||||
if '_id_' in obj_id:
|
||||
obj_id = '.'.join(obj_id.split('_')[:-2]) + ': ' + obj_id.split('_')[-1]
|
||||
obj_id = ('.'.join(obj_id.split('_')[:-2]) + ': ' +
|
||||
obj_id.split('_')[-1])
|
||||
if ': .' in msg and not rel_id:
|
||||
rel_id = msg[msg.find(': .')+3:]
|
||||
if '_id_' in rel_id:
|
||||
rel_id = '.'.join(rel_id.split('_')[:-2]) + ': ' + rel_id.split('_')[-1]
|
||||
rel_id = ('.'.join(rel_id.split('_')[:-2]) +
|
||||
': ' + rel_id.split('_')[-1])
|
||||
msg = msg[:msg.find(': .')]
|
||||
log['last_log'].append('%s|%s\t|%s\t|%s' % (level.ljust(5), obj_id, rel_id, msg))
|
||||
log['last_log'].append('%s|%s\t|%s\t|%s' % (level.ljust(5),
|
||||
obj_id, rel_id, msg))
|
||||
_logger.debug(data)
|
||||
cols = list(flds) # copy to avoid side effects
|
||||
errmsg = str()
|
||||
if table_obj.raise_import_errors:
|
||||
model_obj.import_data(cr, uid, cols, [data], noupdate=table_obj.noupdate)
|
||||
model_obj.import_data(cr, uid, cols, [data],
|
||||
noupdate=table_obj.noupdate)
|
||||
else:
|
||||
try:
|
||||
model_obj.import_data(cr, uid, cols, [data], noupdate=table_obj.noupdate)
|
||||
model_obj.import_data(cr, uid, cols, [data],
|
||||
noupdate=table_obj.noupdate)
|
||||
except:
|
||||
errmsg = str(sys.exc_info()[1])
|
||||
if errmsg and not table_obj.ignore_rel_errors:
|
||||
|
|
@ -98,16 +116,19 @@ class import_odbc_dbtable(orm.Model):
|
|||
# Warn and retry ignoring many2one fields...
|
||||
append_to_log(log, 'WARN', data, errmsg)
|
||||
log['last_warn_count'] += 1
|
||||
# Try ignoring each many2one (tip: in the SQL sentence select more problematic FKs first)
|
||||
# Try ignoring each many2one
|
||||
# (tip: in the SQL sentence select more problematic FKs first)
|
||||
i = find_m2o(cols)
|
||||
if i >= 0:
|
||||
# Try again without the [i] column
|
||||
del cols[i]
|
||||
del data[i]
|
||||
self._import_data(cr, uid, cols, data, model_obj, table_obj, log)
|
||||
self._import_data(cr, uid, cols, data, model_obj,
|
||||
table_obj, log)
|
||||
else:
|
||||
# Fail
|
||||
append_to_log(log, 'ERROR', data, 'Removed all m2o keys and still fails.')
|
||||
append_to_log(log, 'ERROR', data,
|
||||
'Removed all m2o keys and still fails.')
|
||||
log['last_error_count'] += 1
|
||||
return False
|
||||
return True
|
||||
|
|
@ -123,10 +144,12 @@ class import_odbc_dbtable(orm.Model):
|
|||
if not obj.enabled:
|
||||
continue # skip
|
||||
|
||||
_logger.setLevel(obj.raise_import_errors and logging.DEBUG or _loglvl)
|
||||
_logger.setLevel(obj.raise_import_errors and
|
||||
logging.DEBUG or _loglvl)
|
||||
_logger.debug('Importing %s...' % obj.name)
|
||||
|
||||
# now() microseconds are stripped to avoid problem with SQL smalldate
|
||||
# now() microseconds are stripped
|
||||
# to avoid problem with SQL smalldate
|
||||
# TODO: convert UTC Now to local timezone
|
||||
# http://stackoverflow.com/questions/4770297/python-convert-utc-datetime-string-to-local-datetime
|
||||
model_name = obj.model_target.model
|
||||
|
|
@ -150,15 +173,19 @@ class import_odbc_dbtable(orm.Model):
|
|||
obj.sql_source, params, metadata=True)
|
||||
|
||||
# Exclude columns titled "None"; add (xml_)"id" column
|
||||
cidx = [i for i, x in enumerate(res['cols']) if x.upper() != 'NONE']
|
||||
cols = [x for i, x in enumerate(res['cols']) if x.upper() != 'NONE'] + ['id']
|
||||
cidx = ([i for i, x in enumerate(res['cols'])
|
||||
if x.upper() != 'NONE'])
|
||||
cols = ([x for i, x in enumerate(res['cols'])
|
||||
if x.upper() != 'NONE'] + ['id'])
|
||||
|
||||
# Import each row:
|
||||
for row in res['rows']:
|
||||
# Build data row; import only columns present in the "cols" list
|
||||
# Build data row;
|
||||
# import only columns present in the "cols" list
|
||||
data = list()
|
||||
for i in cidx:
|
||||
# TODO: Handle imported datetimes properly - convert from localtime to UTC!
|
||||
# TODO: Handle imported datetimes properly
|
||||
# convert from localtime to UTC!
|
||||
v = row[i]
|
||||
if isinstance(v, str):
|
||||
v = v.strip()
|
||||
|
|
@ -169,7 +196,8 @@ class import_odbc_dbtable(orm.Model):
|
|||
log['last_record_count'] += 1
|
||||
self._import_data(cr, uid, cols, data, model_obj, obj, log)
|
||||
if log['last_record_count'] % 500 == 0:
|
||||
_logger.info('...%s rows processed...' % (log['last_record_count']))
|
||||
_logger.info('...%s rows processed...'
|
||||
% (log['last_record_count']))
|
||||
|
||||
# Finished importing all rows
|
||||
# If no errors, write new sync date
|
||||
|
|
@ -180,12 +208,16 @@ class import_odbc_dbtable(orm.Model):
|
|||
level = logging.WARN
|
||||
if log['last_error_count']:
|
||||
level = logging.ERROR
|
||||
_logger.log(level, 'Imported %s , %d rows, %d errors, %d warnings.' % (
|
||||
model_name, log['last_record_count'], log['last_error_count'],
|
||||
_logger.log(level,
|
||||
'Imported %s , %d rows, %d errors, %d warnings.' %
|
||||
(model_name, log['last_record_count'],
|
||||
log['last_error_count'],
|
||||
log['last_warn_count']))
|
||||
# Write run log, either if the table import is active or inactive
|
||||
if log['last_log']:
|
||||
log['last_log'].insert(0, 'LEVEL|== Line == |== Relationship ==|== Message ==')
|
||||
log['last_log'].insert(0,
|
||||
'LEVEL|== Line == |== Relationship \
|
||||
==|== Message ==')
|
||||
log.update({'last_log': '\n'.join(log['last_log'])})
|
||||
log.update({'last_run': datetime.now().replace(microsecond=0)})
|
||||
self.write(cr, uid, [obj.id], log)
|
||||
|
|
@ -7,30 +7,41 @@
|
|||
<record model="ir.ui.view" id="view_import_dbtable_form">
|
||||
<field name="name">import.odbc.dbtable.form</field>
|
||||
<field name="model">import.odbc.dbtable</field>
|
||||
<field name="type">form</field>
|
||||
<field name="arch" type="xml">
|
||||
<form string="Table">
|
||||
<group>
|
||||
<group>
|
||||
<field name="name" search="1"/>
|
||||
<field name="exec_order"/>
|
||||
<field name="model_target"/>
|
||||
<field name="dbsource_id" search="1"/>
|
||||
<field name="noupdate"/>
|
||||
<field name="enabled"/>
|
||||
<field name="ignore_rel_errors"/>
|
||||
<field name="raise_import_errors"/>
|
||||
<field name="last_sync"/>
|
||||
<group colspan="2">
|
||||
<button name="import_run" string="Run Import" type="object" icon="gtk-execute"/>
|
||||
</group>
|
||||
<group>
|
||||
<field name="exec_order"/>
|
||||
<field name="dbsource_id" search="1"/>
|
||||
<field name="enabled"/>
|
||||
<field name="raise_import_errors"/>
|
||||
<button name="import_schedule" string="Schedule Import" type="object" icon="gtk-paste"/>
|
||||
</group>
|
||||
</group>
|
||||
<group>
|
||||
<field name="sql_source" colspan="4"/>
|
||||
</group>
|
||||
<separator string="Last execution" colspan="4"/>
|
||||
<group>
|
||||
<group>
|
||||
<field name="last_record_count"/>
|
||||
<field name="start_run"/>
|
||||
<field name="last_warn_count"/>
|
||||
<field name="last_run"/>
|
||||
<field name="last_error_count"/>
|
||||
</group>
|
||||
<group>
|
||||
<field name="start_run"/>
|
||||
<field name="last_run"/>
|
||||
<field name="last_log" colspan="4"/>
|
||||
</group>
|
||||
</group>
|
||||
</form>
|
||||
</field>
|
||||
</record>
|
||||
Loading…
Reference in New Issue