From 5cc37f95b1949b7a6395ce8b0b183ca31daa20b9 Mon Sep 17 00:00:00 2001 From: Nicolas Seinlet Date: Tue, 3 Oct 2017 14:17:19 +0200 Subject: [PATCH 1/6] session_db : store sessions in a database rather than in filestore --- session_db/__init__.py | 1 + session_db/__manifest__.py | 24 +++++++ session_db/models/__init__.py | 1 + session_db/models/session.py | 125 ++++++++++++++++++++++++++++++++++ 4 files changed, 151 insertions(+) create mode 100644 session_db/__init__.py create mode 100644 session_db/__manifest__.py create mode 100644 session_db/models/__init__.py create mode 100644 session_db/models/session.py diff --git a/session_db/__init__.py b/session_db/__init__.py new file mode 100644 index 000000000..bff786c08 --- /dev/null +++ b/session_db/__init__.py @@ -0,0 +1 @@ +import models diff --git a/session_db/__manifest__.py b/session_db/__manifest__.py new file mode 100644 index 000000000..d805980ff --- /dev/null +++ b/session_db/__manifest__.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +{ + 'name': "Store sessions in DB", + 'description': """ + Storing sessions in DB +- workls only with workers > 0 +- set the session_db parameter in the odoo config file +- session_db parameter value is a full postgresql connection string, like user:passwd@server/db +- choose another DB than the odoo db itself, for security purpose +- it also possible to use another PostgreSQL user for the same security reasons + +Set this module in the server wide modules + """, + 'category': '', + 'version': '1.0', + + 'depends': [ + ], + + 'data': [ + ], + 'demo': [ + ], +} diff --git a/session_db/models/__init__.py b/session_db/models/__init__.py new file mode 100644 index 000000000..ca2be93e7 --- /dev/null +++ b/session_db/models/__init__.py @@ -0,0 +1 @@ +import session diff --git a/session_db/models/session.py b/session_db/models/session.py new file mode 100644 index 000000000..3bd96d352 --- /dev/null +++ b/session_db/models/session.py @@ -0,0 +1,125 @@ +# -*- coding: utf-8 -*- +import psycopg2 +import json +import logging +import random +import werkzeug.contrib.sessions +import time + +import odoo +from odoo import http +from odoo.tools.func import lazy_property + +_logger = logging.getLogger(__name__) + +def with_cursor(func): + def wrapper(self, *args, **kwargs): + tries = 0 + while True: + tries += 1 + try: + return func(self, *args, **kwargs) + except psycopg2.InterfaceError as e: + _logger.info("Session in DB connection Retry %s/5" % tries) + if tries>4: + raise e + self._open_connection() + return wrapper + +class PGSessionStore(werkzeug.contrib.sessions.SessionStore): + # FIXME This class is NOT thread-safe. Only use in worker mode + def __init__(self, uri, session_class=None): + super(PGSessionStore, self).__init__(session_class) + self._uri = uri + self._open_connection() + self._setup_db() + + def __del__(self): + self._cr.close() + + def _open_connection(self): + cnx = odoo.sql_db.db_connect(self._uri, allow_uri=True) + self._cr = cnx.cursor() + self._cr.autocommit(True) + + @with_cursor + def _setup_db(self): + self._cr.execute(""" + CREATE TABLE IF NOT EXISTS http_sessions ( + sid varchar PRIMARY KEY, + write_date timestamp without time zone NOT NULL, + payload text NOT NULL + ) + """) + + @with_cursor + def save(self, session): + payload = json.dumps(dict(session)) + self._cr.execute(""" + INSERT INTO http_sessions(sid, write_date, payload) + VALUES (%(sid)s, now() at time zone 'UTC', %(payload)s) + ON CONFLICT (sid) + DO UPDATE SET payload = %(payload)s, + write_date = now() at time zone 'UTC' + """, dict(sid=session.sid, payload=payload)) + + @with_cursor + def delete(self, session): + self._cr.execute("DELETE FROM http_sessions WHERE sid=%s", [session.sid]) + + @with_cursor + def get(self, sid): + self._cr.execute("UPDATE http_sessions SET write_date = now() at time zone 'UTC' WHERE sid=%s", [sid]) + self._cr.execute("SELECT payload FROM http_sessions WHERE sid=%s", [sid]) + try: + data = json.loads(self._cr.fetchone()[0]) + except Exception: + return self.new() + + return self.session_class(data, sid, False) + + @with_cursor + def gc(self): + self._cr.execute( + "DELETE FROM http_sessions WHERE now() at time zone 'UTC' - write_date > '7 days'" + ) + + +def session_gc(session_store): + """ + Global cleaning of sessions using either the standard way (delete session files), + Or the DB way. + """ + if random.random() < 0.001: + # we keep session one week + if hasattr(session_store, 'gc'): + session_store.gc() + return + last_week = time.time() - 60*60*24*7 + for fname in os.listdir(session_store.path): + path = os.path.join(session_store.path, fname) + try: + if os.path.getmtime(path) < last_week: + os.unlink(path) + except OSError: + pass + +class Root(http.Root): + @lazy_property + def session_store(self): + """ + Store sessions in DB rather than on FS if parameter permit so + """ + # Setup http sessions + session_db = odoo.tools.config.get('session_db') + if session_db: + _logger.debug("Sessions in db %s" % session_db) + return PGSessionStore(session_db, session_class=http.OpenERPSession) + path = odoo.tools.config.session_dir + _logger.debug('HTTP sessions stored in: %s', path) + return werkzeug.contrib.sessions.FilesystemSessionStore(path, session_class=http.OpenERPSession) + +# #Monkey patch of standard methods +_logger.debug("Monkey patching sessions") +http.session_gc = session_gc +http.root = Root() From 6ff5bb8e03a4e550d98744f608162601d6a21b80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 29 Oct 2022 14:31:47 +0200 Subject: [PATCH 2/6] [MIG] session_db to 16.0 --- session_db/README.rst | 85 ++++++++++++++++ session_db/__init__.py | 2 +- session_db/__manifest__.py | 27 +---- session_db/models/__init__.py | 1 - session_db/models/session.py | 125 ----------------------- session_db/pg_session_store.py | 129 ++++++++++++++++++++++++ session_db/readme/DESCRIPTION.rst | 1 + session_db/readme/ROADMAP.rst | 1 + session_db/readme/USAGE.rst | 7 ++ setup/session_db/odoo/addons/session_db | 1 + setup/session_db/setup.py | 6 ++ 11 files changed, 236 insertions(+), 149 deletions(-) create mode 100644 session_db/README.rst delete mode 100644 session_db/models/__init__.py delete mode 100644 session_db/models/session.py create mode 100644 session_db/pg_session_store.py create mode 100644 session_db/readme/DESCRIPTION.rst create mode 100644 session_db/readme/ROADMAP.rst create mode 100644 session_db/readme/USAGE.rst create mode 120000 setup/session_db/odoo/addons/session_db create mode 100644 setup/session_db/setup.py diff --git a/session_db/README.rst b/session_db/README.rst new file mode 100644 index 000000000..413eea272 --- /dev/null +++ b/session_db/README.rst @@ -0,0 +1,85 @@ +==================== +Store sessions in DB +==================== + +.. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png + :target: https://odoo-community.org/page/development-status + :alt: Beta +.. |badge2| image:: https://img.shields.io/badge/licence-LGPL--3-blue.png + :target: http://www.gnu.org/licenses/lgpl-3.0-standalone.html + :alt: License: LGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fserver--tools-lightgray.png?logo=github + :target: https://github.com/OCA/server-tools/tree/16.0/session_db + :alt: OCA/server-tools +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/server-tools-16-0/server-tools-16-0-session_db + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runbot-Try%20me-875A7B.png + :target: https://runbot.odoo-community.org/runbot/149/16.0 + :alt: Try me on Runbot + +|badge1| |badge2| |badge3| |badge4| |badge5| + +Store sessions in a database instead of the filesystem. + +**Table of contents** + +.. contents:: + :local: + +Usage +===== + +Set this module in the server wide modules. + +Set a ``SESSION_DB_URI`` environment variable as a full postgresql connection string, +like ``postgres://user:passwd@server/db`` or ``db``. + +It is recommended to use a dedicated database for this module, and possibly a dedicated +postgres user for additional security. + +Known issues / Roadmap +====================== + +This module does not work with multi-threaded workers, so it requires workers > 0. + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us smashing it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +~~~~~~~ + +* Odoo SA +* ACSONE SA/NV + +Maintainers +~~~~~~~~~~~ + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +This module is part of the `OCA/server-tools `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/session_db/__init__.py b/session_db/__init__.py index bff786c08..d051c561c 100644 --- a/session_db/__init__.py +++ b/session_db/__init__.py @@ -1 +1 @@ -import models +from . import pg_session_store diff --git a/session_db/__manifest__.py b/session_db/__manifest__.py index d805980ff..c644d2679 100644 --- a/session_db/__manifest__.py +++ b/session_db/__manifest__.py @@ -1,24 +1,7 @@ -# -*- coding: utf-8 -*- { - 'name': "Store sessions in DB", - 'description': """ - Storing sessions in DB -- workls only with workers > 0 -- set the session_db parameter in the odoo config file -- session_db parameter value is a full postgresql connection string, like user:passwd@server/db -- choose another DB than the odoo db itself, for security purpose -- it also possible to use another PostgreSQL user for the same security reasons - -Set this module in the server wide modules - """, - 'category': '', - 'version': '1.0', - - 'depends': [ - ], - - 'data': [ - ], - 'demo': [ - ], + "name": "Store sessions in DB", + "version": "16.0.1.0.0", + "author": "Odoo SA,ACSONE SA/NV,Odoo Community Association (OCA)", + "license": "LGPL-3", + "website": "https://github.com/OCA/server-tools", } diff --git a/session_db/models/__init__.py b/session_db/models/__init__.py deleted file mode 100644 index ca2be93e7..000000000 --- a/session_db/models/__init__.py +++ /dev/null @@ -1 +0,0 @@ -import session diff --git a/session_db/models/session.py b/session_db/models/session.py deleted file mode 100644 index 3bd96d352..000000000 --- a/session_db/models/session.py +++ /dev/null @@ -1,125 +0,0 @@ -# -*- coding: utf-8 -*- -import psycopg2 -import json -import logging -import random -import werkzeug.contrib.sessions -import time - -import odoo -from odoo import http -from odoo.tools.func import lazy_property - -_logger = logging.getLogger(__name__) - -def with_cursor(func): - def wrapper(self, *args, **kwargs): - tries = 0 - while True: - tries += 1 - try: - return func(self, *args, **kwargs) - except psycopg2.InterfaceError as e: - _logger.info("Session in DB connection Retry %s/5" % tries) - if tries>4: - raise e - self._open_connection() - return wrapper - -class PGSessionStore(werkzeug.contrib.sessions.SessionStore): - # FIXME This class is NOT thread-safe. Only use in worker mode - def __init__(self, uri, session_class=None): - super(PGSessionStore, self).__init__(session_class) - self._uri = uri - self._open_connection() - self._setup_db() - - def __del__(self): - self._cr.close() - - def _open_connection(self): - cnx = odoo.sql_db.db_connect(self._uri, allow_uri=True) - self._cr = cnx.cursor() - self._cr.autocommit(True) - - @with_cursor - def _setup_db(self): - self._cr.execute(""" - CREATE TABLE IF NOT EXISTS http_sessions ( - sid varchar PRIMARY KEY, - write_date timestamp without time zone NOT NULL, - payload text NOT NULL - ) - """) - - @with_cursor - def save(self, session): - payload = json.dumps(dict(session)) - self._cr.execute(""" - INSERT INTO http_sessions(sid, write_date, payload) - VALUES (%(sid)s, now() at time zone 'UTC', %(payload)s) - ON CONFLICT (sid) - DO UPDATE SET payload = %(payload)s, - write_date = now() at time zone 'UTC' - """, dict(sid=session.sid, payload=payload)) - - @with_cursor - def delete(self, session): - self._cr.execute("DELETE FROM http_sessions WHERE sid=%s", [session.sid]) - - @with_cursor - def get(self, sid): - self._cr.execute("UPDATE http_sessions SET write_date = now() at time zone 'UTC' WHERE sid=%s", [sid]) - self._cr.execute("SELECT payload FROM http_sessions WHERE sid=%s", [sid]) - try: - data = json.loads(self._cr.fetchone()[0]) - except Exception: - return self.new() - - return self.session_class(data, sid, False) - - @with_cursor - def gc(self): - self._cr.execute( - "DELETE FROM http_sessions WHERE now() at time zone 'UTC' - write_date > '7 days'" - ) - - -def session_gc(session_store): - """ - Global cleaning of sessions using either the standard way (delete session files), - Or the DB way. - """ - if random.random() < 0.001: - # we keep session one week - if hasattr(session_store, 'gc'): - session_store.gc() - return - last_week = time.time() - 60*60*24*7 - for fname in os.listdir(session_store.path): - path = os.path.join(session_store.path, fname) - try: - if os.path.getmtime(path) < last_week: - os.unlink(path) - except OSError: - pass - -class Root(http.Root): - @lazy_property - def session_store(self): - """ - Store sessions in DB rather than on FS if parameter permit so - """ - # Setup http sessions - session_db = odoo.tools.config.get('session_db') - if session_db: - _logger.debug("Sessions in db %s" % session_db) - return PGSessionStore(session_db, session_class=http.OpenERPSession) - path = odoo.tools.config.session_dir - _logger.debug('HTTP sessions stored in: %s', path) - return werkzeug.contrib.sessions.FilesystemSessionStore(path, session_class=http.OpenERPSession) - -# #Monkey patch of standard methods -_logger.debug("Monkey patching sessions") -http.session_gc = session_gc -http.root = Root() diff --git a/session_db/pg_session_store.py b/session_db/pg_session_store.py new file mode 100644 index 000000000..05a46514b --- /dev/null +++ b/session_db/pg_session_store.py @@ -0,0 +1,129 @@ +# Copyright (c) Odoo SA 2017 +# @author Nicolas Seinlet +# Copyright (c) ACSONE SA 2022 +# @author Stéphane Bidoul +import json +import logging +import os + +import psycopg2 + +import odoo +from odoo import http +from odoo.tools._vendor import sessions +from odoo.tools.func import lazy_property + +_logger = logging.getLogger(__name__) + + +def with_cursor(func): + def wrapper(self, *args, **kwargs): + tries = 0 + while True: + tries += 1 + try: + return func(self, *args, **kwargs) + except psycopg2.InterfaceError as e: + _logger.info("Session in DB connection Retry %s/5" % tries) + if tries > 4: + raise e + self._open_connection() + + return wrapper + + +class PGSessionStore(sessions.SessionStore): + def __init__(self, uri, session_class=None): + super().__init__(session_class) + self._uri = uri + self._cr = None + # FIXME This class is NOT thread-safe. Only use in worker mode + if odoo.tools.config["workers"] == 0: + raise ValueError("session_db requires multiple workers") + self._open_connection() + self._setup_db() + + def __del__(self): + if self._cr is not None: + self._cr.close() + + def _open_connection(self): + cnx = odoo.sql_db.db_connect(self._uri, allow_uri=True) + self._cr = cnx.cursor() + self._cr._cnx.autocommit = True + + @with_cursor + def _setup_db(self): + self._cr.execute( + """ + CREATE TABLE IF NOT EXISTS http_sessions ( + sid varchar PRIMARY KEY, + write_date timestamp without time zone NOT NULL, + payload text NOT NULL + ) + """ + ) + + @with_cursor + def save(self, session): + payload = json.dumps(dict(session)) + self._cr.execute( + """ + INSERT INTO http_sessions(sid, write_date, payload) + VALUES (%(sid)s, now() at time zone 'UTC', %(payload)s) + ON CONFLICT (sid) + DO UPDATE SET payload = %(payload)s, + write_date = now() at time zone 'UTC' + """, + dict(sid=session.sid, payload=payload), + ) + + @with_cursor + def delete(self, session): + self._cr.execute("DELETE FROM http_sessions WHERE sid=%s", (session.sid,)) + + @with_cursor + def get(self, sid): + self._cr.execute( + "UPDATE http_sessions " + "SET write_date = now() at time zone 'UTC' " + "WHERE sid=%s", + [sid], + ) + self._cr.execute("SELECT payload FROM http_sessions WHERE sid=%s", (sid,)) + try: + data = json.loads(self._cr.fetchone()[0]) + except Exception: + return self.new() + + return self.session_class(data, sid, False) + + # This method is not part of the Session interface but is called nevertheless, + # so let's get it from FilesystemSessionStore. + rotate = http.FilesystemSessionStore.rotate + + @with_cursor + def vacuum(self): + self._cr.execute( + "DELETE FROM http_sessions " + "WHERE now() at time zone 'UTC' - write_date > '7 days'" + ) + + +_original_session_store = http.root.__class__.session_store + + +@lazy_property +def session_store(self): + session_db_uri = os.environ.get("SESSION_DB_URI") + if session_db_uri: + _logger.debug("HTTP sessions stored in: db") + return PGSessionStore(session_db_uri, session_class=http.Session) + return _original_session_store.__get__(self, self.__class__) + + +# Monkey patch of standard methods +_logger.debug("Monkey patching session store") +http.root.__class__.session_store = session_store +# Reset the lazy property cache +vars(http.root).pop("session_store", None) diff --git a/session_db/readme/DESCRIPTION.rst b/session_db/readme/DESCRIPTION.rst new file mode 100644 index 000000000..2b129a053 --- /dev/null +++ b/session_db/readme/DESCRIPTION.rst @@ -0,0 +1 @@ +Store sessions in a database instead of the filesystem. diff --git a/session_db/readme/ROADMAP.rst b/session_db/readme/ROADMAP.rst new file mode 100644 index 000000000..c4cedb7ba --- /dev/null +++ b/session_db/readme/ROADMAP.rst @@ -0,0 +1 @@ +This module does not work with multi-threaded workers, so it requires workers > 0. diff --git a/session_db/readme/USAGE.rst b/session_db/readme/USAGE.rst new file mode 100644 index 000000000..8ea69803a --- /dev/null +++ b/session_db/readme/USAGE.rst @@ -0,0 +1,7 @@ +Set this module in the server wide modules. + +Set a ``SESSION_DB_URI`` environment variable as a full postgresql connection string, +like ``postgres://user:passwd@server/db`` or ``db``. + +It is recommended to use a dedicated database for this module, and possibly a dedicated +postgres user for additional security. diff --git a/setup/session_db/odoo/addons/session_db b/setup/session_db/odoo/addons/session_db new file mode 120000 index 000000000..0028a0bcc --- /dev/null +++ b/setup/session_db/odoo/addons/session_db @@ -0,0 +1 @@ +../../../../session_db \ No newline at end of file diff --git a/setup/session_db/setup.py b/setup/session_db/setup.py new file mode 100644 index 000000000..28c57bb64 --- /dev/null +++ b/setup/session_db/setup.py @@ -0,0 +1,6 @@ +import setuptools + +setuptools.setup( + setup_requires=['setuptools-odoo'], + odoo_addon=True, +) From b68adf53a2d73eaf32ab9cfa2f70ef3c77eb35b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 29 Oct 2022 15:36:13 +0200 Subject: [PATCH 3/6] session_db: use SESSION_LIFETIME constant --- session_db/pg_session_store.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/session_db/pg_session_store.py b/session_db/pg_session_store.py index 05a46514b..951b67a22 100644 --- a/session_db/pg_session_store.py +++ b/session_db/pg_session_store.py @@ -106,7 +106,8 @@ class PGSessionStore(sessions.SessionStore): def vacuum(self): self._cr.execute( "DELETE FROM http_sessions " - "WHERE now() at time zone 'UTC' - write_date > '7 days'" + "WHERE now() at time zone 'UTC' - write_date > %s", + (f"{http.SESSION_LIFETIME} seconds",), ) From 42f7db536a9b824f531fa1807476f50cf1807860 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 29 Oct 2022 15:38:03 +0200 Subject: [PATCH 4/6] session_db: do not update write_date on get The upstream FilesystemSessionStore does not do that. --- session_db/pg_session_store.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/session_db/pg_session_store.py b/session_db/pg_session_store.py index 951b67a22..3feae5f2b 100644 --- a/session_db/pg_session_store.py +++ b/session_db/pg_session_store.py @@ -84,12 +84,6 @@ class PGSessionStore(sessions.SessionStore): @with_cursor def get(self, sid): - self._cr.execute( - "UPDATE http_sessions " - "SET write_date = now() at time zone 'UTC' " - "WHERE sid=%s", - [sid], - ) self._cr.execute("SELECT payload FROM http_sessions WHERE sid=%s", (sid,)) try: data = json.loads(self._cr.fetchone()[0]) From 30ccee3d450e68d82dbb263f3478f935a82121ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sat, 29 Oct 2022 17:39:22 +0200 Subject: [PATCH 5/6] session_db: declare maintainer --- session_db/__manifest__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/session_db/__manifest__.py b/session_db/__manifest__.py index c644d2679..e969d96f2 100644 --- a/session_db/__manifest__.py +++ b/session_db/__manifest__.py @@ -4,4 +4,5 @@ "author": "Odoo SA,ACSONE SA/NV,Odoo Community Association (OCA)", "license": "LGPL-3", "website": "https://github.com/OCA/server-tools", + "maintainers": ["sbidoul"], } From 0f7f6f5c143622b7419745285a4f27dacee77a52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Tue, 17 Jan 2023 16:50:59 +0100 Subject: [PATCH 6/6] session_db: explain why such as module is useful --- session_db/readme/DESCRIPTION.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/session_db/readme/DESCRIPTION.rst b/session_db/readme/DESCRIPTION.rst index 2b129a053..ee705201a 100644 --- a/session_db/readme/DESCRIPTION.rst +++ b/session_db/readme/DESCRIPTION.rst @@ -1 +1,3 @@ -Store sessions in a database instead of the filesystem. +Store sessions in a database instead of the filesystem. This simplifies the +configuration of horizontally scalable deployments, by avoiding the need for a +distributed filesystem to store the Odoo sessions.