From 10eb3ae56974b76de5ceba8664f1acbbd426e1a1 Mon Sep 17 00:00:00 2001 From: Marc Durepos Date: Tue, 28 May 2024 10:13:43 -0400 Subject: [PATCH 01/11] caldav_sync: add video conference and alarms sync --- caldav_sync/__manifest__.py | 2 +- caldav_sync/models/calendar_event.py | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/caldav_sync/__manifest__.py b/caldav_sync/__manifest__.py index 0ad599a..3c65a92 100644 --- a/caldav_sync/__manifest__.py +++ b/caldav_sync/__manifest__.py @@ -8,7 +8,7 @@ { 'name': 'CalDAV Synchronization', - 'version': '17.0.0.5.2', + 'version': '17.0.0.5.3', 'license': 'LGPL-3', 'category': 'Productivity', 'summary': 'Synchronize Odoo Calendar Events with CalDAV Servers', diff --git a/caldav_sync/models/calendar_event.py b/caldav_sync/models/calendar_event.py index 44de0f9..f2a18cb 100644 --- a/caldav_sync/models/calendar_event.py +++ b/caldav_sync/models/calendar_event.py @@ -152,7 +152,21 @@ class CalendarEvent(models.Model): ical_event.add('description', event.description) if event.location: ical_event.add('location', event.location) - + if event.videocall_location: + ical_event.add('CONFERENCE', event.videocall_location) + for partner in event.partner_ids: + attendee = vCalAddress(f'MAILTO:{partner.email}') + attendee.params['cn'] = vText(partner.name) + attendee_record = self.env['calendar.attendee'].search([('event_id', '=', event.id), ('partner_id', '=', partner.id)], limit=1) + if attendee_record: + attendee.params['partstat'] = vText(self._map_attendee_status(attendee_record.state)) + ical_event.add('attendee', attendee, encode=0) + for alarm in event.alarm_ids: + ical_alarm = Alarm() + ical_alarm.add('trigger', alarm.trigger) + ical_alarm.add('action', 'DISPLAY') + ical_alarm.add('description', alarm.name or 'Reminder') + ical_event.add_component(ical_alarm) # Add RRULE if the event is recurrent if event.recurrency and event.recurrence_id: rrule = event.recurrence_id._get_rrule() @@ -310,6 +324,7 @@ class CalendarEvent(models.Model): 'stop': end, 'description': str(component.get('description')), 'location': str(component.get('location')), + 'videocall_location': str(component.get('conference')), 'caldav_uid': uid, 'partner_ids': [(6, 0, attendee_ids.ids)], } From f61d759737fb01a66645e642b96b041b8676f57a Mon Sep 17 00:00:00 2001 From: Marc Durepos Date: Tue, 28 May 2024 10:20:11 -0400 Subject: [PATCH 02/11] remove alarm sync (it doesn't work) --- caldav_sync/models/calendar_event.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/caldav_sync/models/calendar_event.py b/caldav_sync/models/calendar_event.py index f2a18cb..0e0b3f3 100644 --- a/caldav_sync/models/calendar_event.py +++ b/caldav_sync/models/calendar_event.py @@ -3,7 +3,7 @@ from odoo import models, api, fields import caldav import logging from datetime import datetime -from icalendar import Calendar, Event, vCalAddress, Alarm, vText, vWeekday +from icalendar import Calendar, Event, vCalAddress, vText, vWeekday from bs4 import BeautifulSoup from datetime import timedelta import dateutil @@ -161,12 +161,6 @@ class CalendarEvent(models.Model): if attendee_record: attendee.params['partstat'] = vText(self._map_attendee_status(attendee_record.state)) ical_event.add('attendee', attendee, encode=0) - for alarm in event.alarm_ids: - ical_alarm = Alarm() - ical_alarm.add('trigger', alarm.trigger) - ical_alarm.add('action', 'DISPLAY') - ical_alarm.add('description', alarm.name or 'Reminder') - ical_event.add_component(ical_alarm) # Add RRULE if the event is recurrent if event.recurrency and event.recurrence_id: rrule = event.recurrence_id._get_rrule() From babd69fad995d9afc9de1fe06cdeed5916905a3d Mon Sep 17 00:00:00 2001 From: Marc Durepos Date: Tue, 28 May 2024 10:30:56 -0400 Subject: [PATCH 03/11] get description text as text again --- caldav_sync/models/calendar_event.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/caldav_sync/models/calendar_event.py b/caldav_sync/models/calendar_event.py index 0e0b3f3..2c3672c 100644 --- a/caldav_sync/models/calendar_event.py +++ b/caldav_sync/models/calendar_event.py @@ -148,8 +148,8 @@ class CalendarEvent(models.Model): ical_event.add('dtstamp', event.write_date.replace(tzinfo=None)) if event.name: ical_event.add('summary', event.name) - if event.description: - ical_event.add('description', event.description) + if self.html_to_text(event.description): + ical_event.add('description', self.html_to_text(event.description)) if event.location: ical_event.add('location', event.location) if event.videocall_location: @@ -312,13 +312,14 @@ class CalendarEvent(models.Model): end = component.decoded('dtend') if isinstance(end, datetime): end = end.replace(tzinfo=None) + values = { 'name': str(component.get('summary')), 'start': start, 'stop': end, - 'description': str(component.get('description')), - 'location': str(component.get('location')), - 'videocall_location': str(component.get('conference')), + 'description': self._extract_component_text(component, 'description'), + 'location': self._extract_component_text(component, 'location'), + 'videocall_location': self._extract_component_text(component, 'conference'), 'caldav_uid': uid, 'partner_ids': [(6, 0, attendee_ids.ids)], } @@ -336,6 +337,10 @@ class CalendarEvent(models.Model): 'follow_recurrence': True, }) existing_instance.with_context({'caldav_no_sync': True}).write(values) + @staticmethod + def _extract_component_text(component, subcomponent_name): + text = str(component.get(subcomponent_name)) + text = text if text != 'None' else '' @staticmethod From f6d6e37c27cfc437e9ccf7ad486cdb0ad9649a69 Mon Sep 17 00:00:00 2001 From: Marc Durepos Date: Tue, 28 May 2024 10:58:03 -0400 Subject: [PATCH 04/11] caldav_sync: try to fix tz problems --- caldav_sync/models/calendar_event.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/caldav_sync/models/calendar_event.py b/caldav_sync/models/calendar_event.py index 2c3672c..e9d00db 100644 --- a/caldav_sync/models/calendar_event.py +++ b/caldav_sync/models/calendar_event.py @@ -2,12 +2,13 @@ import uuid from odoo import models, api, fields import caldav import logging -from datetime import datetime +from datetime import datetime, timezone from icalendar import Calendar, Event, vCalAddress, vText, vWeekday from bs4 import BeautifulSoup from datetime import timedelta import dateutil import re +from pytz import timezone _logger = logging.getLogger(__name__) @@ -143,9 +144,12 @@ class CalendarEvent(models.Model): calendar.add('version', '2.0') for event in self: + user_tz = timezone('UTC') + if event.user_id.tz: + user_tz = timezone(event.user_id.tz) ical_event = Event() ical_event.add('uid', event.caldav_uid) - ical_event.add('dtstamp', event.write_date.replace(tzinfo=None)) + ical_event.add('dtstamp', event.write_date.astimezone(user_tz)) if event.name: ical_event.add('summary', event.name) if self.html_to_text(event.description): @@ -168,8 +172,8 @@ class CalendarEvent(models.Model): ical_event.add('rrule', rrule_dict) # Add DTSTART and DTEND - ical_event.add('dtstart', event.start.replace(tzinfo=None)) - ical_event.add('dtend', event.stop.replace(tzinfo=None)) + ical_event.add('dtstart', event.start) + ical_event.add('dtend', event.stop) calendar.add_component(ical_event) @@ -279,6 +283,9 @@ class CalendarEvent(models.Model): email_regex = re.compile(r'[a-z0-9\.\-+_]+@[a-z0-9\.\-+_]+\.[a-z]+') now = datetime.now() current_user_email = self.env.user.email.lower() + user_tz = timezone('UTC') + if self.env.user.tz: + user_tz = timezone(self.env.user.tz) for component in ical_event.subcomponents: if isinstance(component, Event): @@ -308,10 +315,10 @@ class CalendarEvent(models.Model): existing_instance = self._get_existing_instance(uid, recurrence_id) start = component.decoded('dtstart') if isinstance(start, datetime): - start = start.replace(tzinfo=None) + start = start.astimezone(user_tz).replace(tzinfo=None) end = component.decoded('dtend') if isinstance(end, datetime): - end = end.replace(tzinfo=None) + end = end.astimezone(user_tz).replace(tzinfo=None) values = { 'name': str(component.get('summary')), From 599a3975e8111f2fcd79c1c220472323777e7804 Mon Sep 17 00:00:00 2001 From: Marc Durepos Date: Tue, 28 May 2024 11:00:25 -0400 Subject: [PATCH 05/11] fix html_to_text typo --- caldav_sync/models/calendar_event.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/caldav_sync/models/calendar_event.py b/caldav_sync/models/calendar_event.py index e9d00db..caacf1d 100644 --- a/caldav_sync/models/calendar_event.py +++ b/caldav_sync/models/calendar_event.py @@ -152,8 +152,8 @@ class CalendarEvent(models.Model): ical_event.add('dtstamp', event.write_date.astimezone(user_tz)) if event.name: ical_event.add('summary', event.name) - if self.html_to_text(event.description): - ical_event.add('description', self.html_to_text(event.description)) + if self._html_to_text(event.description): + ical_event.add('description', self._html_to_text(event.description)) if event.location: ical_event.add('location', event.location) if event.videocall_location: From b0be821fa37c4437f6ca5a9ba439d8b3d0b7dbab Mon Sep 17 00:00:00 2001 From: Marc Durepos Date: Tue, 28 May 2024 11:05:10 -0400 Subject: [PATCH 06/11] attempt to fix tz issue --- caldav_sync/models/calendar_event.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/caldav_sync/models/calendar_event.py b/caldav_sync/models/calendar_event.py index caacf1d..d08238c 100644 --- a/caldav_sync/models/calendar_event.py +++ b/caldav_sync/models/calendar_event.py @@ -172,8 +172,8 @@ class CalendarEvent(models.Model): ical_event.add('rrule', rrule_dict) # Add DTSTART and DTEND - ical_event.add('dtstart', event.start) - ical_event.add('dtend', event.stop) + ical_event.add('dtstart', event.start.astimezone(user_tz)) + ical_event.add('dtend', event.stop.astimezone(user_tz)) calendar.add_component(ical_event) From 308e2ea9a90b0cbe35d7d9df36ad813db91a761d Mon Sep 17 00:00:00 2001 From: Marc Durepos Date: Tue, 28 May 2024 11:15:49 -0400 Subject: [PATCH 07/11] attempt to fix tz issue --- caldav_sync/models/calendar_event.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/caldav_sync/models/calendar_event.py b/caldav_sync/models/calendar_event.py index d08238c..0ca437a 100644 --- a/caldav_sync/models/calendar_event.py +++ b/caldav_sync/models/calendar_event.py @@ -8,7 +8,7 @@ from bs4 import BeautifulSoup from datetime import timedelta import dateutil import re -from pytz import timezone +from pytz import timezone, utc _logger = logging.getLogger(__name__) @@ -149,7 +149,7 @@ class CalendarEvent(models.Model): user_tz = timezone(event.user_id.tz) ical_event = Event() ical_event.add('uid', event.caldav_uid) - ical_event.add('dtstamp', event.write_date.astimezone(user_tz)) + ical_event.add('dtstamp', utc.localize(event.write_date).astimezone(user_tz)) if event.name: ical_event.add('summary', event.name) if self._html_to_text(event.description): @@ -172,8 +172,8 @@ class CalendarEvent(models.Model): ical_event.add('rrule', rrule_dict) # Add DTSTART and DTEND - ical_event.add('dtstart', event.start.astimezone(user_tz)) - ical_event.add('dtend', event.stop.astimezone(user_tz)) + ical_event.add('dtstart', utc.localize(event.start).astimezone(user_tz)) + ical_event.add('dtend', utc.localize(event.stop).astimezone(user_tz)) calendar.add_component(ical_event) @@ -281,11 +281,7 @@ class CalendarEvent(models.Model): def sync_event_from_ical(self, ical_event): email_regex = re.compile(r'[a-z0-9\.\-+_]+@[a-z0-9\.\-+_]+\.[a-z]+') - now = datetime.now() current_user_email = self.env.user.email.lower() - user_tz = timezone('UTC') - if self.env.user.tz: - user_tz = timezone(self.env.user.tz) for component in ical_event.subcomponents: if isinstance(component, Event): @@ -315,10 +311,10 @@ class CalendarEvent(models.Model): existing_instance = self._get_existing_instance(uid, recurrence_id) start = component.decoded('dtstart') if isinstance(start, datetime): - start = start.astimezone(user_tz).replace(tzinfo=None) + start = start.astimezone(utc).replace(tzinfo=None) end = component.decoded('dtend') if isinstance(end, datetime): - end = end.astimezone(user_tz).replace(tzinfo=None) + end = end.astimezone(utc).replace(tzinfo=None) values = { 'name': str(component.get('summary')), @@ -344,12 +340,12 @@ class CalendarEvent(models.Model): 'follow_recurrence': True, }) existing_instance.with_context({'caldav_no_sync': True}).write(values) + @staticmethod def _extract_component_text(component, subcomponent_name): text = str(component.get(subcomponent_name)) text = text if text != 'None' else '' - @staticmethod def _html_to_text(html): return BeautifulSoup(html, "html.parser").getText() From f4a44c24eced83e5ad2ae0d7b75f2cfab0bec2a3 Mon Sep 17 00:00:00 2001 From: Marc Durepos Date: Tue, 28 May 2024 11:34:53 -0400 Subject: [PATCH 08/11] finally got attendees and timezones working well --- caldav_sync/__manifest__.py | 3 +++ caldav_sync/models/calendar_event.py | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/caldav_sync/__manifest__.py b/caldav_sync/__manifest__.py index 3c65a92..fbed16b 100644 --- a/caldav_sync/__manifest__.py +++ b/caldav_sync/__manifest__.py @@ -20,6 +20,9 @@ 'author': 'Bemade Inc.', 'website': 'https://www.bemade.org', 'depends': ['base', 'calendar'], + 'external_dependencies': { + 'python': ['caldav', 'icalendar', 'bs4'], + }, 'images': ['static/description/images/main_screenshot.png'], 'data': [ 'views/res_users_views.xml', diff --git a/caldav_sync/models/calendar_event.py b/caldav_sync/models/calendar_event.py index 0ca437a..f228952 100644 --- a/caldav_sync/models/calendar_event.py +++ b/caldav_sync/models/calendar_event.py @@ -152,19 +152,24 @@ class CalendarEvent(models.Model): ical_event.add('dtstamp', utc.localize(event.write_date).astimezone(user_tz)) if event.name: ical_event.add('summary', event.name) - if self._html_to_text(event.description): + if event.description and self._html_to_text(event.description): ical_event.add('description', self._html_to_text(event.description)) if event.location: ical_event.add('location', event.location) if event.videocall_location: ical_event.add('CONFERENCE', event.videocall_location) for partner in event.partner_ids: + if partner == event.user_id.partner_id: + continue attendee = vCalAddress(f'MAILTO:{partner.email}') attendee.params['cn'] = vText(partner.name) attendee_record = self.env['calendar.attendee'].search([('event_id', '=', event.id), ('partner_id', '=', partner.id)], limit=1) if attendee_record: attendee.params['partstat'] = vText(self._map_attendee_status(attendee_record.state)) ical_event.add('attendee', attendee, encode=0) + organizer = vCalAddress(f"MAILTO:{event.user_id.email}") + organizer.params['cn'] = event.user_id.name + ical_event.add('organizer', organizer) # Add RRULE if the event is recurrent if event.recurrency and event.recurrence_id: rrule = event.recurrence_id._get_rrule() From 9733d63bb7415a3b491ab54ee8a0fedb5248373e Mon Sep 17 00:00:00 2001 From: Marc Durepos Date: Wed, 29 May 2024 10:02:41 -0400 Subject: [PATCH 09/11] attempt fix for recurring email blasts to attendees on the hour --- caldav_sync/models/calendar_event.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/caldav_sync/models/calendar_event.py b/caldav_sync/models/calendar_event.py index f228952..295f900 100644 --- a/caldav_sync/models/calendar_event.py +++ b/caldav_sync/models/calendar_event.py @@ -6,7 +6,6 @@ from datetime import datetime, timezone from icalendar import Calendar, Event, vCalAddress, vText, vWeekday from bs4 import BeautifulSoup from datetime import timedelta -import dateutil import re from pytz import timezone, utc @@ -320,7 +319,6 @@ class CalendarEvent(models.Model): end = component.decoded('dtend') if isinstance(end, datetime): end = end.astimezone(utc).replace(tzinfo=None) - values = { 'name': str(component.get('summary')), 'start': start, @@ -339,12 +337,27 @@ class CalendarEvent(models.Model): self.with_context({'caldav_no_sync': True}).create(values) else: _logger.info(f"Updating with vals: {values}") - if recurrency_vals and recurrency_vals.get('recurrency'): + changed_vals = {} + # Don't update partner_ids if no change + if attendee_ids - existing_instance.partner_ids: + changed_vals.update({ + 'partner_ids',values.pop('partner_ids'), + }) + + # Don't write values that haven't changed + for key, val in values.items(): + if getattr(existing_instance, key) != val: + changed_vals.update({key: values.pop(key)}) + if (recurrency_vals and recurrency_vals.get('recurrency') + and (not existing_instance.recurrency or not + existing_instance.follow_recurrence)): existing_instance.write({ 'recurrency': True, 'follow_recurrence': True, }) - existing_instance.with_context({'caldav_no_sync': True}).write(values) + existing_instance.with_context({ + 'caldav_no_sync': True, + }).write(changed_vals) @staticmethod def _extract_component_text(component, subcomponent_name): From 2cb6f3ec69bcceb5c226e181ddda5df890a7e56c Mon Sep 17 00:00:00 2001 From: Marc Durepos Date: Wed, 29 May 2024 10:48:48 -0400 Subject: [PATCH 10/11] Revert "bemade_odoo_partner_scrapper_js_only moved to bemade-tools" This reverts commit ca348b11a209931d2462911c11522cc0fb3b8ead. --- .../__init__.py | 1 + .../__manifest__.py | 33 +++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 bemade_odoo_partner_scrapper_js_only/__init__.py create mode 100644 bemade_odoo_partner_scrapper_js_only/__manifest__.py diff --git a/bemade_odoo_partner_scrapper_js_only/__init__.py b/bemade_odoo_partner_scrapper_js_only/__init__.py new file mode 100644 index 0000000..40a96af --- /dev/null +++ b/bemade_odoo_partner_scrapper_js_only/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/bemade_odoo_partner_scrapper_js_only/__manifest__.py b/bemade_odoo_partner_scrapper_js_only/__manifest__.py new file mode 100644 index 0000000..5214f00 --- /dev/null +++ b/bemade_odoo_partner_scrapper_js_only/__manifest__.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +{ + 'name': 'Bemade Odoo Partner Scrapper JS Only', + 'version': '17.0.1.0.0', + 'category': 'Administration', + 'summary': 'Module for scraping partners from odoo.com.', + 'description': """ + This module enables the scraping of partners from odoo.com. It allows for the collection and management of partner contact information. + + Main Features: + * Automatically scrape partners from odoo.com. + * Extracts partner name, website, and contact information. + * Manage contact information of partners. + """, + 'license': 'GPL-3', + 'author': 'Bemade', + 'website': 'https://www.bemade.org', + 'depends': [ + 'bemade_odoo_partner_scrapper' + ], + 'data': [ + ], + "assets": { + "web.assets_backend": [ + "bemade_odoo_partner_scrapper/static/src/js/odoo_scrapper.js", + "bemade_odoo_partner_scrapper/static/src/xml/odoo_scrapper_templates.xml", + ], + }, + 'demo': [], + 'installable': True, + 'application': False, + 'auto_install': False +} From 7dc4c05a0a87a95dca445d1394402108666a94ad Mon Sep 17 00:00:00 2001 From: Marc Durepos Date: Wed, 29 May 2024 10:51:02 -0400 Subject: [PATCH 11/11] Revert "bemade_odoo_partner_scrapper moved to bemade-tools" This reverts commit 540e7f17d3e573d2e241deee4a325e2a88044b3d. --- bemade_odoo_partner_scrapper/__init__.py | 3 + bemade_odoo_partner_scrapper/__manifest__.py | 36 +++ bemade_odoo_partner_scrapper/data/demo.xml | 106 +++++++ .../data/res_partner_relation_type.xml | 37 +++ .../models/__init__.py | 3 + .../models/res_partner.py | 264 ++++++++++++++++++ .../static/src/js/odoo_scrapper.js | 68 +++++ .../src/xml/odoo_scrapper_templates.xml | 20 ++ .../views/res_partner_views.xml | 86 ++++++ 9 files changed, 623 insertions(+) create mode 100644 bemade_odoo_partner_scrapper/__init__.py create mode 100644 bemade_odoo_partner_scrapper/__manifest__.py create mode 100644 bemade_odoo_partner_scrapper/data/demo.xml create mode 100644 bemade_odoo_partner_scrapper/data/res_partner_relation_type.xml create mode 100644 bemade_odoo_partner_scrapper/models/__init__.py create mode 100644 bemade_odoo_partner_scrapper/models/res_partner.py create mode 100644 bemade_odoo_partner_scrapper/static/src/js/odoo_scrapper.js create mode 100644 bemade_odoo_partner_scrapper/static/src/xml/odoo_scrapper_templates.xml create mode 100644 bemade_odoo_partner_scrapper/views/res_partner_views.xml diff --git a/bemade_odoo_partner_scrapper/__init__.py b/bemade_odoo_partner_scrapper/__init__.py new file mode 100644 index 0000000..5305644 --- /dev/null +++ b/bemade_odoo_partner_scrapper/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +from . import models \ No newline at end of file diff --git a/bemade_odoo_partner_scrapper/__manifest__.py b/bemade_odoo_partner_scrapper/__manifest__.py new file mode 100644 index 0000000..4cbe3a7 --- /dev/null +++ b/bemade_odoo_partner_scrapper/__manifest__.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +{ + 'name': 'Bemade Odoo Partner Scrapper', + 'version': '17.0.0.0.0', + 'category': 'Administration', + 'summary': 'Module for scraping partners from odoo.com.', + 'description': """ + This module enables the scraping of partners from odoo.com. It allows for the collection and management of partner contact information. + + Main Features: + * Automatically scrape partners from odoo.com. + * Extracts partner name, website, and contact information. + * Manage contact information of partners. + """, + 'sequence': 10, + 'license': 'AGPL-3', + 'author': 'Bemade', + 'website': 'https://www.bemade.org', + 'depends': ['base', 'contacts', 'sale_management', 'sale' , 'partner_multi_relation'], + 'data': [ + # No security + 'views/res_partner_views.xml', + # Commented for migration to 17, raises error on update + 'data/res_partner_relation_type.xml' + ], + # "assets": { + # "web.assets_backend": [ + # "bemade_odoo_partner_scrapper/static/src/js/odoo_scrapper.js", + # "bemade_odoo_partner_scrapper/static/src/xml/odoo_scrapper_templates.xml", + # ], + # }, + 'demo': [], + 'installable': True, + 'application': False, + 'auto_install': False +} diff --git a/bemade_odoo_partner_scrapper/data/demo.xml b/bemade_odoo_partner_scrapper/data/demo.xml new file mode 100644 index 0000000..c47c023 --- /dev/null +++ b/bemade_odoo_partner_scrapper/data/demo.xml @@ -0,0 +1,106 @@ + + + + + + Is assistant of + Has assistant + p + p + + + Is competitor of + Is competitor of + c + c + + + + Has worked for + Has former employee + p + c + + + + Washing Companies + + + Washing Gold + + + + Washing Silver + + + + Washing Services + + + + + Great Washing Powder Company + + 1 + Le Bourget du Lac + 73377 + +33 4 49 23 44 54 + + 93, Press Avenue + great@yourcompany.example.com + http://www.great.com + + + Best Washing Powder Company + + 1 + Champs sur Marne + 77420 + + best@yourcompany.example.com + +33 1 64 61 04 01 + 12 rue Albert Einstein + http://www.best.com/ + + + Super Washing Powder Company + + 1 + 3rd Floor, Room 3-C, + Carretera Panamericana, Km 1, Urb. Delgado Chalbaud + Caracas + 1090 + super@yourcompany.example.com + +58 212 681 0538 + + super.com + + + + + + + + + + + + + + + + + + diff --git a/bemade_odoo_partner_scrapper/data/res_partner_relation_type.xml b/bemade_odoo_partner_scrapper/data/res_partner_relation_type.xml new file mode 100644 index 0000000..fe644ad --- /dev/null +++ b/bemade_odoo_partner_scrapper/data/res_partner_relation_type.xml @@ -0,0 +1,37 @@ + + + + + + Is Odoo Partner Of + Is Odoo Client Of + + + + + + + + Odoo Partner + + + Learning Partner + + + + Ready Partner + + + + Silver Partner + + + + Gold Partner + + + diff --git a/bemade_odoo_partner_scrapper/models/__init__.py b/bemade_odoo_partner_scrapper/models/__init__.py new file mode 100644 index 0000000..cd5f4d8 --- /dev/null +++ b/bemade_odoo_partner_scrapper/models/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +from . import res_partner \ No newline at end of file diff --git a/bemade_odoo_partner_scrapper/models/res_partner.py b/bemade_odoo_partner_scrapper/models/res_partner.py new file mode 100644 index 0000000..b991d0d --- /dev/null +++ b/bemade_odoo_partner_scrapper/models/res_partner.py @@ -0,0 +1,264 @@ +from odoo import models, fields, api, _ +import requests +from bs4 import BeautifulSoup +import logging +from base64 import b64encode +import re + +_logger = logging.getLogger(__name__) + + +class ResPartner(models.Model): + _inherit = 'res.partner' + + is_odoo_partner = fields.Boolean(string="Is Odoo Partner", default=False, tracking=True) + is_odoo_user = fields.Boolean(string="Is Odoo User", default=False, tracking=True) + odoo_name = fields.Char(string="Name on Odoo") + odoo_note = fields.Html(string="Note on Odoo") + odoo_id = fields.Char(string="ID on Odoo") + odoo_url = fields.Char(string="Odoo Partner URL", tracking=True, index=True) + odoo_page = fields.Html(string="Odoo Partner/Referral Page") + odoo_page_update = fields.Date(string="Odoo Page Update") + odoo_partner_type = fields.Selection( + [('learning', 'Learning'), + ('ready', 'Ready'), + ('silver', 'Silver'), + ('gold', 'Gold')], + string="Partner Type", + tracking=True) + + @api.model + def change_color_on_kanban(self): + for record in self: + color = 0 + if record.odoo_partner_type == 'learning': + color = 2 + elif record.odoo_partner_type == 'ready': + color = 10 + elif record.odoo_partner_type == 'silver': + color = 7 + elif record.odoo_partner_type == 'gold': + color = 3 + else: + if record.is_odoo_user: + color = 4 + record.color = color + + color = fields.Integer('Color Index', compute="change_color_on_kanban") + + def set_image_from_url(self, url): + response = requests.get(url) + if response.status_code == 200: + self.image_1920 = b64encode(response.content).decode('utf-8') + _logger.info(f"Image for {self.name} set from {url}") + else: + raise ValueError('Could not download image: got response code {}'.format(response.status_code)) + + @api.model + def get_odoo_partner(self): + + def extract_client_data(soup): + odoo_name = soup.find(id='partner_name').text + odoo_note = soup.find('div', class_='col-lg-8 mt32') + + main_image = soup.find('img', {'class': 'img img-fluid d-block mx-auto mb16'}) + image_link = main_image['src'] + + exist_client = self.env['res.partner'].search([('odoo_name', '=', odoo_name)]) + + if not exist_client: + exist_client = exist_client.create({ + 'name': odoo_name, + 'is_company': True, + 'odoo_name':odoo_name, + 'odoo_note':odoo_note, + 'is_odoo_user': True, + 'odoo_page': soup, + }) + exist_client.set_image_from_url(image_link) + _logger.info(f"Client {exist_client.name} created") + + return exist_client + + def process_partner_relation(partner, client): + if partner.id != client.id: + self.env['res.partner.relation'].create({ + 'left_partner_id' : partner.id, + 'right_partner_id' : client.id, + 'type_id' : self.env.ref('bemade_odoo_partner_scrapper.rel_type_odoo_partner').id + }) + _logger.info(f"Partner {partner.name} Odoo Partner of {client.name}") + else: + _logger.info(f"Partner {partner.name} is the same as client {client.name}") + + def extract_partner_data(url): + page = requests.get("https://www.odoo.com" + url) + soup = BeautifulSoup(page.content, 'html.parser') + + partner_data = {} + + partner_data['name'] = soup.find(id="partner_name").text + partner_type = soup.find('h3', class_='col-lg-12 text-center text-muted') + + if partner_type and partner_type.span: + partner_data['odoo_partner_type'] = partner_type.span.text.strip().lower() + + partner_data['odoo_name'] = soup.find(id="partner_name").text + address_div = soup.find('span', itemprop='streetAddress') + email_div = soup.find('span', itemprop='email') + phone_div = soup.find('span', itemprop='telephone') + website_div = soup.find('span', itemprop='website') + + main_image = soup.find('img', {'class': 'img img-fluid d-block mx-auto mb16'}) + image_link = main_image['src'] + + clients_url = set() + for link in soup.find_all('a'): + href = link.get('href') + if href is not None: + # Ignore any URL containing 'country/canada-36' + if '/customers/' in href: + # Add to set. Duplicates will be ignored automatically. + clients_url.add(href) + + state_data = None + + if address_div is not None: + # Extract address components + address_parts = address_div.get_text(separator="\n").split("\n") + partner_data['street'] = address_parts[0] + city_state_postal = address_parts[len(address_parts) - 2].split(', ') + + if city_state_postal != [',']: + partner_data['city'] = city_state_postal[0] + state_postal = city_state_postal[1].split(' ') + if len(state_postal) == 3 and len(state_postal[2]) == 3: + partner_data['zip'] = state_postal[1] + state_postal[2] + state_data = state_postal[0] + elif len(state_postal) == 2 and state_postal[1]: + if len(state_postal[1]) == 6: + partner_data['zip'] = state_postal[1] + state_data = state_postal[0] + if len(state_postal[1]) == 3: + if len(state_postal[0]) == 2: + partner_data['zip'] = state_postal[1] + state_data = state_postal[0] + else: + partner_data['zip'] = state_postal[0] + state_postal[1] + else: + if len(state_postal[0]) == 6: + partner_data['zip'] = state_postal[0] + else: + state_data = state_postal[0] + + # partner_data['country'] = address_parts[len(address_parts) - 1] + + partner_data['email'] = email_div.string if email_div and email_div.string else False + partner_data['phone'] = phone_div.string if phone_div and phone_div.string else False + partner_data['website'] = website_div.string if website_div and website_div.string else False + partner_data['is_odoo_partner'] = True + partner_data['is_odoo_user'] = True + partner_data['is_company'] = True + partner_data['image_link'] = image_link + partner_data['clients_url'] = clients_url + + if state_data: + country_id = self.env['res.country'].search([('code', '=', 'CA')]).id + partner_data['state_id'] = self.env['res.country.state'].search([('code', '=', state_data),('country_id', '=', country_id)]).id + + partner_data = {k: v for k, v in partner_data.items() if v is not False} + + return partner_data + + # Load the webpage at the given url + page = requests.get("https://www.odoo.com/fr_FR/partners/country/canada-36") + # Parse the HTML content + soup = BeautifulSoup(page.content, 'html.parser') + div = soup.find(id="ref_content") + partners_url = set() + other_pages = set() + + for link in div.find_all('a'): + href = link.get('href') + if href is not None: + # Ignore any URL containing 'country/canada-36' + if '/page/' in href: + # Add to set. Duplicates will be ignored automatically. + other_pages.add(href) + + for link in div.find_all('a'): + href = link.get('href') + if href is not None and href != '': + # Ignore any URL containing 'country/canada-36' + if 'country/canada-36' not in href: + # Add to set. Duplicates will be ignored automatically. + partners_url.add(href.split('#', 1)[0].split('?', 1)[0]) + + for other_page in other_pages: + page = requests.get("https://www.odoo.com" + other_page) + soup = BeautifulSoup(page.content, 'html.parser') + div = soup.find(id="ref_content") + for link in div.find_all('a'): + href = link.get('href') + if href is not None and href != '': + # Ignore any URL containing 'country/canada-36' + if 'country/canada-36' not in href: + # Add to set. Duplicates will be ignored automatically. + partners_url.add(href.split('#', 1)[0].split('?', 1)[0]) + + for partner_url in partners_url: + # page = requests.get("https://www.odoo.com" + partner_url) + # soup = BeautifulSoup(page.content, 'html.parser') + + odoo_partner = extract_partner_data(partner_url) + _logger.info(f"Processing {odoo_partner['name']}") + + image_url = False + if odoo_partner['image_link']: + image_url = odoo_partner['image_link'] + del odoo_partner['image_link'] + _logger.info(f"Image for {odoo_partner['name']} set from {image_url}") + + if odoo_partner['clients_url']: + clients_url = odoo_partner['clients_url'] + else: + clients_url = set() + + if 'clients_url' in odoo_partner: + del odoo_partner['clients_url'] + + exist_odoo_name = self.env['res.partner'].search([('odoo_name', '=', odoo_partner['odoo_name'])]) + if exist_odoo_name: + _logger.info(f"Partner Odoo Name {odoo_partner['name']} already exist") + new_partner = exist_odoo_name + else: + exist_email = self.env['res.partner'].search([('email', '=', odoo_partner['email'])]) if 'email' in odoo_partner else False + if exist_email: + _logger.info(f"Partner Email {odoo_partner['name']} already exist") + new_partner = exist_email + else: + exist_phone = self.env['res.partner'].search([('phone', '=', odoo_partner['phone'])]) if 'phone' in odoo_partner else False + if exist_phone: + _logger.info(f"Partner Phone {odoo_partner['name']} already exist") + new_partner = exist_phone + else: + _logger.info(f"Partner {odoo_partner['name']} created") + new_partner = self.env['res.partner'].create(odoo_partner) + if image_url: + new_partner.set_image_from_url(image_url) + + for client_url in clients_url: + if 'http' not in client_url: + exist_url = self.env['res.partner'].search([('odoo_url', '=', client_url)]) + # add condition on odoo_date, actually not pulling data if date exist + if exist_url and exist_url.odoo_page: + soup = exist_url.odoo_page + _logger.info(f"Client {exist_url.name} reloading soup from db") + else: + print("https://www.odoo.com" + client_url) + page = requests.get("https://www.odoo.com" + client_url) + soup = BeautifulSoup(page.content, 'html.parser') + _logger.info(f"Client {exist_url.name} read from odoo.com") + + client = extract_client_data(soup) + process_partner_relation(new_partner, client) diff --git a/bemade_odoo_partner_scrapper/static/src/js/odoo_scrapper.js b/bemade_odoo_partner_scrapper/static/src/js/odoo_scrapper.js new file mode 100644 index 0000000..cd7b9ef --- /dev/null +++ b/bemade_odoo_partner_scrapper/static/src/js/odoo_scrapper.js @@ -0,0 +1,68 @@ +/** @odoo-module **/ + +import ListController from 'web.ListController'; +import ListView from 'web.ListView'; + +import KanbanController from 'web.KanbanController'; +import KanbanView from 'web.KanbanView'; + +const viewRegistry = require('web.view_registry'); + +const OdooScrapperListController = ListController.extend({ + // buttons_template must match the t-name on the template for the button (static xml) + buttons_template: 'odoo_scrapper.list_view_buttons', + events: _.extend({}, ListController.prototype.events, { + 'click .o_button_get_partner': '_onGetPartnerClick', + }), + // This may need to be async function() if there needs to be an await this._rpc({ ... }); call to not reload early + _onGetPartnerClick: function () { + this._rpc({ + model: 'res.partner', + method: 'get_odoo_partner', + args: [], + }).then(() => { + this.reload(); + }); + // Couldn't test this for real, but it runs the action. May need a this.reload() + }, +}); + +export const OdooScrapperListView = ListView.extend({ + config: _.extend({}, ListView.prototype.config, { + Controller: OdooScrapperListController, + }), +}); + +// key must match with the js_class attribute of the tree view you want to modify +viewRegistry.add('res_partner_odoo_scrapper_tree', OdooScrapperListView); + +const OdooScrapperKanbanController = KanbanController.extend({ + // buttons_template must match the t-name on the template for the button (static xml) + buttons_template: 'odoo_scrapper.kanban_view_buttons', + events: _.extend({}, KanbanController.prototype.events, { + 'click .o_button_get_partner': '_onGetPartnerClick', + }), + // This may need to be async function() if there needs to be an await this._rpc({ ... }); call to not reload early + _onGetPartnerClick: function () { + this._rpc({ + model: 'res.partner', + method: 'get_odoo_partner', + args: [], + }).then(() => { + this.reload(); + }); + // Couldn't test this for real, but it runs the action. May need a this.reload() + }, +}); + +export const OdooScrapperKanbanView = KanbanView.extend({ + config: _.extend({}, KanbanView.prototype.config, { + Controller: OdooScrapperKanbanController, + }), +}); + +// key must match with the js_class attribute of the tree view you want to modify +viewRegistry.add('res_partner_odoo_scrapper_kanban', OdooScrapperKanbanView); + + + diff --git a/bemade_odoo_partner_scrapper/static/src/xml/odoo_scrapper_templates.xml b/bemade_odoo_partner_scrapper/static/src/xml/odoo_scrapper_templates.xml new file mode 100644 index 0000000..af0a1dd --- /dev/null +++ b/bemade_odoo_partner_scrapper/static/src/xml/odoo_scrapper_templates.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/bemade_odoo_partner_scrapper/views/res_partner_views.xml b/bemade_odoo_partner_scrapper/views/res_partner_views.xml new file mode 100644 index 0000000..22402a4 --- /dev/null +++ b/bemade_odoo_partner_scrapper/views/res_partner_views.xml @@ -0,0 +1,86 @@ + + + + + res.partner.form.inherit + res.partner + + + +
+ Odoo User + Odoo Partner +
+
+

Partner

+
+ + + + + + + + + + + + + + + + + + + + res.partner.tree.inherit + res.partner + + + + res_partner_odoo_scrapper_tree + + + + + + + + + + + res.partner.kanban.inherit + res.partner + + + + res_partner_odoo_scrapper_kanban + + + + + + oe_kanban_global_click o_kanban_record_has_image_fill o_res_partner_kanban oe_kanban_color_#{record.color} + + + + + + + + + + + res.partner.odoo.search + res.partner + + + + + + + + + +