Techsystech
/
server-tools
mirror of https://github.com/OCA/server-tools.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[FIX] PEP8 compliance and review comments

pull/4/head
Maxime Chambreuil 2013-09-07 15:39:36 -04:00
parent 7e09d76422
commit 0042904f45
6 changed files with 244 additions and 223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
base_external_dbsource/__init__.py
View File

@ -19,6 +19,6 @@
#
##############################################################################
import base_external_dbsource
from . import base_external_dbsource
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

3
base_external_dbsource/__openerp__.py
View File

@ -27,7 +27,7 @@
This module allows you to define connections to foreign databases using ODBC,
Oracle Client or SQLAlchemy.
Databases sources can be configured in Settings > Configuration -> Data sources.
Database sources can be configured in Settings > Configuration -> Data sources.
Depending on the database, you need:
* to install unixodbc and python-pyodbc packages to use ODBC connections.
@ -44,7 +44,6 @@ Depending on the database, you need:
'depends': [
'base',
],
'init': [],
'data': [
'base_external_dbsource_view.xml',
'security/ir.model.access.csv',

93
base_external_dbsource/base_external_dbsource.py
View File

@ -20,10 +20,10 @@
##############################################################################
import os
from osv import fields, osv
import logging
from openerp.osv import orm, fields
from openerp.tools.translate import _
import openerp.tools as tools
import logging
_logger = logging.getLogger(__name__)
CONNECTORS = []
@ -31,56 +31,67 @@ CONNECTORS = []
try:
import sqlalchemy
import pymssql
CONNECTORS.append( ('mssql', 'Microsoft SQL Server') )
CONNECTORS.append(('mssql', 'Microsoft SQL Server'))
except:
_logger.info('MS SQL Server not available. Please install "slqalchemy" and "pymssql" python package.')
_logger.info('MS SQL Server not available. Please install "slqalchemy"\
and "pymssql" python package.')
try:
import sqlalchemy
import MySQLdb
CONNECTORS.append( ('mysql', 'MySQL') )
CONNECTORS.append(('mysql', 'MySQL'))
except:
_logger.info('MySQL not available. Please install "slqalchemy" and "mysqldb" python package.')
_logger.info('MySQL not available. Please install "slqalchemy" and\
"mysqldb" python package.')
try:
import pyodbc
CONNECTORS.append( ('pyodbc', 'ODBC') )
CONNECTORS.append(('pyodbc', 'ODBC'))
except:
_logger.info('ODBC libraries not available. Please install "unixodbc" and "python-pyodbc" packages.')
_logger.info('ODBC libraries not available. Please install "unixodbc"\
and "python-pyodbc" packages.')
try:
import cx_Oracle
CONNECTORS.append( ('cx_Oracle', 'Oracle') )
CONNECTORS.append(('cx_Oracle', 'Oracle'))
except:
_logger.info('Oracle libraries not available. Please install "cx_Oracle" python package.')
_logger.info('Oracle libraries not available. Please install "cx_Oracle"\
python package.')
import psycopg2
CONNECTORS.append( ('postgresql', 'PostgreSQL') )
CONNECTORS.append(('postgresql', 'PostgreSQL'))
try:
import sqlalchemy
CONNECTORS.append( ('sqlite', 'SQLite') )
CONNECTORS.append(('sqlite', 'SQLite'))
except:
_logger.info('SQLAlchemy not available. Please install "slqalchemy" python package.')
_logger.info('SQLAlchemy not available. Please install "slqalchemy" python\
package.')
class base_external_dbsource(osv.osv):
class base_external_dbsource(orm.Model):
_name = "base.external.dbsource"
_description = 'External Database Sources'
_columns = {
'name': fields.char('Datasource name', required=True, size=64),
'conn_string': fields.text('Connection string', help="""\
'conn_string': fields.text('Connection string', help="""
Sample connection strings:
- Microsoft SQL Server: mssql+pymssql://username:%s@server:port/dbname?charset=utf8
- Microsoft SQL Server:
mssql+pymssql://username:%s@server:port/dbname?charset=utf8
- MySQL: mysql://user:%s@server:port/dbname
- ODBC: DRIVER={FreeTDS};SERVER=server.address;Database=mydb;UID=sa
- ORACLE: username/%s@//server.address:port/instance
- PostgreSQL: dbname='template1' user='dbuser' host='localhost' port='5432' password=%s
- PostgreSQL:
dbname='template1' user='dbuser' host='localhost' port='5432' password=%s
- SQLite: sqlite:///test.db
"""),
'password': fields.char('Password' , size=40),
'connector': fields.selection(CONNECTORS, 'Connector', required=True,
help = "If a connector is missing from the list, check the " \
+ "server log to confirm that the required componentes were detected."),
'password': fields.char('Password', size=40),
'connector': fields.selection(CONNECTORS, 'Connector',
required=True,
help="If a connector is missing from the\
list, check the server log to confirm\
that the required components were\
detected."),
}
def conn_open(self, cr, uid, id1):
@ -98,24 +109,28 @@ Sample connection strings:
conn = cx_Oracle.connect(connStr)
elif data.connector == 'pyodbc':
conn = pyodbc.connect(connStr)
elif data.connector in ('sqlite','mysql','mssql'):
elif data.connector in ('sqlite', 'mysql', 'mssql'):
conn = sqlalchemy.create_engine(connStr).connect()
elif data.connector == 'postgresql':
conn = psycopg2.connect(connStr)
return conn
def execute(self, cr, uid, ids, sqlquery, sqlparams=None, metadata=False, context=None):
def execute(self, cr, uid, ids, sqlquery, sqlparams=None, metadata=False,
context=None):
"""Executes SQL and returns a list of rows.
"sqlparams" can be a dict of values, that can be referenced in the SQL statement
using "%(key)s" or, in the case of Oracle, ":key".
"sqlparams" can be a dict of values, that can be referenced in
the SQL statement using "%(key)s" or, in the case of Oracle,
":key".
Example:
sqlquery = "select * from mytable where city = %(city)s and date > %(dt)s"
params = {'city': 'Lisbon', 'dt': datetime.datetime(2000, 12, 31)}
sqlquery = "select * from mytable where city = %(city)s and
date > %(dt)s"
params = {'city': 'Lisbon',
'dt': datetime.datetime(2000, 12, 31)}
If metadata=True, it will instead return a dict containing the rows list and the columns list,
in the format:
If metadata=True, it will instead return a dict containing the
rows list and the columns list, in the format:
{ 'cols': [ 'col_a', 'col_b', ...]
, 'rows': [ (a0, b0, ...), (a1, b1, ...), ...] }
"""
@ -123,16 +138,18 @@ Sample connection strings:
rows, cols = list(), list()
for obj in data:
conn = self.conn_open(cr, uid, obj.id)
if obj.connector in ["sqlite","mysql","mssql"]:
if obj.connector in ["sqlite", "mysql", "mssql"]:
#using sqlalchemy
cur = conn.execute(sqlquery, sqlparams)
if metadata: cols = cur.keys()
if metadata:
cols = cur.keys()
rows = [r for r in cur]
else:
#using other db connectors
cur = conn.cursor()
cur.execute(sqlquery, sqlparams)
if metadata: cols = [d[0] for d in cur.description]
if metadata:
cols = [d[0] for d in cur.description]
rows = cur.fetchall()
conn.close()
if metadata:
@ -146,14 +163,18 @@ Sample connection strings:
try:
conn = self.conn_open(cr, uid, obj.id)
except Exception, e:
raise osv.except_osv(_("Connection test failed!"), _("Here is what we got instead:\n %s") % tools.ustr(e))
raise osv.except_osv(_("Connection test failed!"),
_("Here is what we got instead:\n %s")
% tools.ustr(e))
finally:
try:
if conn: conn.close()
if conn:
conn.close()
except Exception:
# ignored, just a consequence of the previous exception
pass
#TODO: if OK a (wizard) message box should be displayed
raise osv.except_osv(_("Connection test succeeded!"), _("Everything seems properly set up!"))
raise osv.except_osv(_("Connection test succeeded!"),
_("Everything seems properly set up!"))
base_external_dbsource()
#EOF

2
import_odbc/__init__.py
View File

@ -19,6 +19,6 @@
#
##############################################################################
import import_odbc
from . import import_odbc
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

1
import_odbc/__openerp__.py
View File

@ -68,7 +68,6 @@ Improvements ideas waiting for a contributor:
'base',
'base_external_dbsource',
],
'init': [],
'data': [
'import_odbc_view.xml',
'security/ir.model.access.csv',

82
import_odbc/import_odbc.py
View File

@ -21,14 +21,15 @@
import sys
from datetime import datetime
from osv import fields, osv
from openerp.osv import orm, fields
import logging
_logger = logging.getLogger(__name__)
_loglvl = _logger.getEffectiveLevel()
SEP = '|'
SEP = '|'
class import_odbc_dbtable(osv.osv):
_name="import.odbc.dbtable"
class import_odbc_dbtable(orm.Model):
_name = "import.odbc.dbtable"
_description = 'Import Table Data'
_order = 'exec_order'
_columns = {
@ -36,7 +37,7 @@ class import_odbc_dbtable(osv.osv):
'enabled': fields.boolean('Execution enabled'),
'dbsource_id': fields.many2one('base.external.dbsource', 'Database source', required=True),
'sql_source': fields.text('SQL', required=True, help='Column names must be valid "import_data" columns.'),
'model_target': fields.many2one('ir.model','Target object'),
'model_target': fields.many2one('ir.model', 'Target object'),
'noupdate': fields.boolean('No updates', help="Only create new records; disable updates to existing records."),
'exec_order': fields.integer('Execution order', help="Defines the order to perform the import"),
'last_sync': fields.datetime('Last sync date', help="Datetime for the last succesfull sync. Later changes on the source may not be replicated on the destination"),
@ -47,10 +48,10 @@ class import_odbc_dbtable(osv.osv):
'last_warn_count': fields.integer('Last warning count', readonly=True),
'last_log': fields.text('Last run log', readonly=True),
'ignore_rel_errors': fields.boolean('Ignore relationship errors',
help = "On error try to reimport rows ignoring relationships."),
help="On error try to reimport rows ignoring relationships."),
'raise_import_errors': fields.boolean('Raise import errors',
help = "Import errors not handled, intended for debugging purposes."
+ "\nAlso forces debug messages to be written to the server log."),
help="Import errors not handled, intended for debugging purposes."
"\nAlso forces debug messages to be written to the server log."),
}
_defaults = {
'enabled': True,
@ -63,11 +64,11 @@ class import_odbc_dbtable(osv.osv):
def find_m2o(field_list):
""""Find index of first column with a one2many field"""
for i, x in enumerate(field_list):
if len(x)>3 and x[-3:] == ':id' or x[-3:] == '/id':
if len(x) > 3 and x[-3:] == ':id' or x[-3:] == '/id':
return i
return -1
def append_to_log(log, level, obj_id = '', msg = '', rel_id = ''):
def append_to_log(log, level, obj_id='', msg='', rel_id=''):
if '_id_' in obj_id:
obj_id = '.'.join(obj_id.split('_')[:-2]) + ': ' + obj_id.split('_')[-1]
if ': .' in msg and not rel_id:
@ -76,10 +77,8 @@ class import_odbc_dbtable(osv.osv):
rel_id = '.'.join(rel_id.split('_')[:-2]) + ': ' + rel_id.split('_')[-1]
msg = msg[:msg.find(': .')]
log['last_log'].append('%s|%s\t|%s\t|%s' % (level.ljust(5), obj_id, rel_id, msg))
_logger.debug( data )
cols = list(flds) #copy to avoid side effects
_logger.debug(data)
cols = list(flds) # copy to avoid side effects
errmsg = str()
if table_obj.raise_import_errors:
model_obj.import_data(cr, uid, cols, [data], noupdate=table_obj.noupdate)
@ -88,15 +87,14 @@ class import_odbc_dbtable(osv.osv):
model_obj.import_data(cr, uid, cols, [data], noupdate=table_obj.noupdate)
except:
errmsg = str(sys.exc_info()[1])
if errmsg and not table_obj.ignore_rel_errors:
#Fail
append_to_log(log, 'ERROR', data, errmsg )
append_to_log(log, 'ERROR', data, errmsg)
log['last_error_count'] += 1
return False
if errmsg and table_obj.ignore_rel_errors:
#Warn and retry ignoring many2one fields...
append_to_log(log, 'WARN', data, errmsg )
append_to_log(log, 'WARN', data, errmsg)
log['last_warn_count'] += 1
#Try ignoring each many2one (tip: in the SQL sentence select more problematic FKs first)
i = find_m2o(cols)
@ -107,32 +105,30 @@ class import_odbc_dbtable(osv.osv):
self._import_data(cr, uid, cols, data, model_obj, table_obj, log)
else:
#Fail
append_to_log(log, 'ERROR', data, 'Removed all m2o keys and still fails.' )
append_to_log(log, 'ERROR', data, 'Removed all m2o keys and still fails.')
log['last_error_count'] += 1
return False
return True
def import_run(self, cr, uid, ids=None, context=None):
db_model = self.pool.get('base.external.dbsource')
actions = self.read(cr, uid, ids, ['id', 'exec_order'])
actions.sort(key = lambda x:(x['exec_order'], x['id']))
actions.sort(key=lambda x: (x['exec_order'], x['id']))
#Consider each dbtable:
for action_ref in actions:
obj = self.browse(cr, uid, action_ref['id'])
if not obj.enabled: continue #skip
if not obj.enabled:
continue # skip
_logger.setLevel(obj.raise_import_errors and logging.DEBUG or _loglvl)
_logger.debug('Importing %s...' % obj.name)
#now() microseconds are stripped to avoid problem with SQL smalldate
#TODO: convert UTC Now to local timezone (http://stackoverflow.com/questions/4770297/python-convert-utc-datetime-string-to-local-datetime)
#TODO: convert UTC Now to local timezone
#http://stackoverflow.com/questions/4770297/python-convert-utc-datetime-string-to-local-datetime
model_name = obj.model_target.model
model_obj = self.pool.get(model_name)
model_obj = self.pool.get(model_name)
xml_prefix = model_name.replace('.', '_') + "_id_"
log = {'start_run': datetime.now().replace(microsecond=0),
'last_run': None,
@ -143,10 +139,13 @@ class import_odbc_dbtable(osv.osv):
self.write(cr, uid, [obj.id], log)
#Prepare SQL sentence; replace "%s" with the last_sync date
if obj.last_sync: sync = datetime.strptime(obj.last_sync, "%Y-%m-%d %H:%M:%S")
else: sync = datetime.datetime(1900, 1, 1, 0, 0, 0)
if obj.last_sync:
sync = datetime.strptime(obj.last_sync, "%Y-%m-%d %H:%M:%S")
else:
sync = datetime.datetime(1900, 1, 1, 0, 0, 0)
params = {'sync': sync}
res = db_model.execute(cr, uid, [obj.dbsource_id.id], obj.sql_source, params, metadata=True)
res = db_model.execute(cr, uid, [obj.dbsource_id.id],
obj.sql_source, params, metadata=True)
#Exclude columns titled "None"; add (xml_)"id" column
cidx = [i for i, x in enumerate(res['cols']) if x.upper() != 'NONE']
@ -159,37 +158,40 @@ class import_odbc_dbtable(osv.osv):
for i in cidx:
#TODO: Handle imported datetimes properly - convert from localtime to UTC!
v = row[i]
if isinstance(v, str): v = v.strip()
if isinstance(v, str):
v = v.strip()
data.append(v)
data.append( xml_prefix + str(row[0]).strip() )
data.append(xml_prefix + str(row[0]).strip())
#Import the row; on error, write line to the log
log['last_record_count'] += 1
self._import_data(cr, uid, cols, data, model_obj, obj, log)
if log['last_record_count'] % 500 == 0:
_logger.info('...%s rows processed...' % (log['last_record_count']) )
_logger.info('...%s rows processed...' % (log['last_record_count']))
#Finished importing all rows
#If no errors, write new sync date
if not (log['last_error_count'] or log['last_warn_count']):
log['last_sync'] = log['start_run']
level = logging.DEBUG
if log['last_warn_count']: level = logging.WARN
if log['last_error_count']: level = logging.ERROR
if log['last_warn_count']:
level = logging.WARN
if log['last_error_count']:
level = logging.ERROR
_logger.log(level, 'Imported %s , %d rows, %d errors, %d warnings.' % (
model_name, log['last_record_count'], log['last_error_count'] ,
log['last_warn_count'] ) )
model_name, log['last_record_count'], log['last_error_count'],
log['last_warn_count']))
#Write run log, either if the table import is active or inactive
if log['last_log']:
log['last_log'].insert(0, 'LEVEL|== Line == |== Relationship ==|== Message ==')
log.update( {'last_log': '\n'.join(log['last_log'])} )
log.update({ 'last_run': datetime.now().replace(microsecond=0) }) #second=0,
log.update({'last_log': '\n'.join(log['last_log'])})
log.update({'last_run': datetime.now().replace(microsecond=0)})
self.write(cr, uid, [obj.id], log)
#Finished
_logger.debug('Import job FINISHED.')
return True
def import_schedule(self, cr, uid, ids, context=None):
cron_obj = self.pool.get('ir.cron')
new_create_id = cron_obj.create(cr, uid, {
@ -211,4 +213,4 @@ class import_odbc_dbtable(osv.osv):
'type': 'ir.actions.act_window',
}
import_odbc_dbtable()
#EOF