From 0ec4e40c26bace2a78047c4b371b6f7e17eb40e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20V=C3=A9zina?= Date: Wed, 12 Jul 2023 07:17:17 -0400 Subject: [PATCH] partner scrapper --- bemade_odoo_partner_scrapper/__init__.py | 3 + bemade_odoo_partner_scrapper/__manifest__.py | 37 ++++ bemade_odoo_partner_scrapper/data/demo.xml | 106 ++++++++++ .../data/res_partner_relation_type.xml | 36 ++++ .../models/__init__.py | 3 + .../models/res_partner.py | 197 ++++++++++++++++++ .../static/src/js/odoo_scrapper.js | 71 +++++++ .../src/xml/odoo_scrapper_templates.xml | 24 +++ .../views/res_partner_views.xml | 41 ++++ 9 files changed, 518 insertions(+) create mode 100644 bemade_odoo_partner_scrapper/__init__.py create mode 100644 bemade_odoo_partner_scrapper/__manifest__.py create mode 100644 bemade_odoo_partner_scrapper/data/demo.xml create mode 100644 bemade_odoo_partner_scrapper/data/res_partner_relation_type.xml create mode 100644 bemade_odoo_partner_scrapper/models/__init__.py create mode 100644 bemade_odoo_partner_scrapper/models/res_partner.py create mode 100644 bemade_odoo_partner_scrapper/static/src/js/odoo_scrapper.js create mode 100644 bemade_odoo_partner_scrapper/static/src/xml/odoo_scrapper_templates.xml create mode 100644 bemade_odoo_partner_scrapper/views/res_partner_views.xml diff --git a/bemade_odoo_partner_scrapper/__init__.py b/bemade_odoo_partner_scrapper/__init__.py new file mode 100644 index 0000000..5305644 --- /dev/null +++ b/bemade_odoo_partner_scrapper/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +from . import models \ No newline at end of file diff --git a/bemade_odoo_partner_scrapper/__manifest__.py b/bemade_odoo_partner_scrapper/__manifest__.py new file mode 100644 index 0000000..9e389e0 --- /dev/null +++ b/bemade_odoo_partner_scrapper/__manifest__.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +{ + 'name': 'Bemade Odoo Partner Scrapper', + 'version': '1.0.0', + 'category': 'Administration', + 'summary': 'Module for scraping partners from odoo.com.', + 'description': """ + This module enables the scraping of partners from odoo.com. It allows for the collection and management of partner contact information. + + Main Features: + * Automatically scrape partners from odoo.com. + * Extracts partner name, website, and contact information. + * Manage contact information of partners. + """, + 'sequence': 10, + 'license': 'GPL-3', + 'author': 'Bemade', + 'website': 'https://www.bemade.org', + 'depends': ['base', 'contacts', 'sale_management', 'sale' , 'partner_multi_relation'], + 'data': [ + # No security + 'views/res_partner_views.xml', + 'data/res_partner_relation_type.xml' + ], + "assets": { + "web.assets_backend": [ + "bemade_odoo_partner_scrapper/static/src/js/odoo_scrapper.js", + ], + "web.assets_qweb": [ + "bemade_odoo_partner_scrapper/static/src/xml/odoo_scrapper_templates.xml", + ], + }, + 'demo': [], + 'installable': True, + 'application': False, + 'auto_install': False +} diff --git a/bemade_odoo_partner_scrapper/data/demo.xml b/bemade_odoo_partner_scrapper/data/demo.xml new file mode 100644 index 0000000..c47c023 --- /dev/null +++ b/bemade_odoo_partner_scrapper/data/demo.xml @@ -0,0 +1,106 @@ + + + + + + Is assistant of + Has assistant + p + p + + + Is competitor of + Is competitor of + c + c + + + + Has worked for + Has former employee + p + c + + + + Washing Companies + + + Washing Gold + + + + Washing Silver + + + + Washing Services + + + + + Great Washing Powder Company + + 1 + Le Bourget du Lac + 73377 + +33 4 49 23 44 54 + + 93, Press Avenue + great@yourcompany.example.com + http://www.great.com + + + Best Washing Powder Company + + 1 + Champs sur Marne + 77420 + + best@yourcompany.example.com + +33 1 64 61 04 01 + 12 rue Albert Einstein + http://www.best.com/ + + + Super Washing Powder Company + + 1 + 3rd Floor, Room 3-C, + Carretera Panamericana, Km 1, Urb. Delgado Chalbaud + Caracas + 1090 + super@yourcompany.example.com + +58 212 681 0538 + + super.com + + + + + + + + + + + + + + + + + + diff --git a/bemade_odoo_partner_scrapper/data/res_partner_relation_type.xml b/bemade_odoo_partner_scrapper/data/res_partner_relation_type.xml new file mode 100644 index 0000000..b193a2d --- /dev/null +++ b/bemade_odoo_partner_scrapper/data/res_partner_relation_type.xml @@ -0,0 +1,36 @@ + + + + + + Is Odoo Partner Of + Is Odoo Client Of + c + c + + + + + Odoo Partner + + + Learning Partner + + + + Ready Partner + + + + Silver Partner + + + + Gold Partner + + + diff --git a/bemade_odoo_partner_scrapper/models/__init__.py b/bemade_odoo_partner_scrapper/models/__init__.py new file mode 100644 index 0000000..cd5f4d8 --- /dev/null +++ b/bemade_odoo_partner_scrapper/models/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +from . import res_partner \ No newline at end of file diff --git a/bemade_odoo_partner_scrapper/models/res_partner.py b/bemade_odoo_partner_scrapper/models/res_partner.py new file mode 100644 index 0000000..8c283f3 --- /dev/null +++ b/bemade_odoo_partner_scrapper/models/res_partner.py @@ -0,0 +1,197 @@ +from odoo import models, fields, api, _ +import requests +from bs4 import BeautifulSoup +import logging + +_logger = logging.getLogger(__name__) + + +class ResPartner(models.Model): + _inherit = 'res.partner' + + is_odoo_partner = fields.Boolean(string="Is Odoo Partner", default=False, tracking=True) + is_odoo_user = fields.Boolean(string="Is Odoo User", default=False, tracking=True) + odoo_name = fields.Char(string="Name on Odoo", tracking=True) + odoo_id = fields.Char(string="ID on Odoo", tracking=True) + odoo_url = fields.Char(string="Odoo Partner URL", tracking=True, index=True) + + @api.model + def get_odoo_partner(self): + + def extract_partner_data(url): + page = requests.get("https://www.odoo.com" + url) + soup = BeautifulSoup(page.content, 'html.parser') + + partner_data = {} + + partner_data['name'] = soup.find(id="partner_name").text + partner_data['odoo_name'] = soup.find(id="partner_name").text + address_div = soup.find('span', itemprop='streetAddress') + email_div = soup.find('span', itemprop='email') + phone_div = soup.find('span', itemprop='telephone') + website_div = soup.find('span', itemprop='website') + + if address_div is not None: + # Extract address components + address_parts = address_div.get_text(separator="\n").split("\n") + partner_data['street'] = address_parts[0] + city_state_postal = address_parts[len(address_parts) - 2].split(', ') + + if city_state_postal != [',']: + partner_data['city'] = city_state_postal[0] + state_postal = city_state_postal[1].split(' ') + if len(state_postal) == 3 and len(state_postal[2]) == 3: + partner_data['zip'] = state_postal[1] + state_postal[2] + state_data = state_postal[0] + elif len(state_postal) == 2 and state_postal[1]: + if len(state_postal[1]) == 6: + partner_data['zip'] = state_postal[1] + state_data = state_postal[0] + if len(state_postal[1]) == 3: + if len(state_postal[0]) == 2: + partner_data['zip'] = state_postal[1] + state_data = state_postal[0] + else: + partner_data['zip'] = state_postal[0] + state_postal[1] + else: + if len(state_postal[0]) == 6: + partner_data['zip'] = state_postal[0] + else: + state_data = state_postal[0] + + # partner_data['country'] = address_parts[len(address_parts) - 1] + + partner_data['email'] = email_div.string if email_div and email_div.string else '' + partner_data['phone'] = phone_div.string if phone_div and phone_div.string else '' + partner_data['website'] = website_div.string if website_div and website_div.string else '' + partner_data['is_odoo_partner'] = True + partner_data['is_odoo_user'] = True + + if state_data: + partner_data['state_id'] = self.env['res.country.state'].search([('code', '=', state_data)]).id + + return partner_data + + # Load the webpage at the given url + page = requests.get("https://www.odoo.com/fr_FR/partners/country/canada-36") + # Parse the HTML content + soup = BeautifulSoup(page.content, 'html.parser') + div = soup.find(id="ref_content") + partners_url = set() + other_pages = set() + + for link in div.find_all('a'): + href = link.get('href') + if href is not None: + # Ignore any URL containing 'country/canada-36' + if '/page/' in href: + # Add to set. Duplicates will be ignored automatically. + other_pages.add(href) + + for link in div.find_all('a'): + href = link.get('href') + if href is not None and href != '': + # Ignore any URL containing 'country/canada-36' + if 'country/canada-36' not in href: + # Add to set. Duplicates will be ignored automatically. + partners_url.add(href.split('#', 1)[0].split('?', 1)[0]) + + for other_page in other_pages: + page = requests.get("https://www.odoo.com" + other_page) + soup = BeautifulSoup(page.content, 'html.parser') + div = soup.find(id="ref_content") + for link in div.find_all('a'): + href = link.get('href') + if href is not None and href != '': + # Ignore any URL containing 'country/canada-36' + if 'country/canada-36' not in href: + # Add to set. Duplicates will be ignored automatically. + partners_url.add(href.split('#', 1)[0].split('?', 1)[0]) + + for partner_url in partners_url: + page = requests.get("https://www.odoo.com" + partner_url) + soup = BeautifulSoup(page.content, 'html.parser') + + odoo_partner = extract_partner_data(partner_url) + print (odoo_partner) + + # partner_name = soup.find(id="partner_name").text + # address_div = soup.find('span', itemprop='streetAddress') + # email_div = soup.find('span', itemprop='email') + # phone_div = soup.find('span', itemprop='telephone') + # website_div = soup.find('span', itemprop='website') + # if address_div is not None: + # # Extract address components + # address_parts = address_div.get_text(separator="\n").split("\n") + # street_address = address_parts[0] + # city_state_postal = address_parts[len(address_parts) - 2].split(', ') + # + # if city_state_postal != [',']: + # city = city_state_postal[0] + # state_postal = city_state_postal[1].split(' ') + # + # if len(state_postal) == 3 and len(state_postal[2]) == 3: + # postal_code = state_postal[1] + state_postal[2] + # state = state_postal[0] + # elif len(state_postal) == 2 and state_postal[1]: + # if len(state_postal[1]) == 6: + # postal_code = state_postal[1] + # state = state_postal[0] + # if len(state_postal[1]) == 3: + # if len(state_postal[0]) == 2: + # postal_code = state_postal[1] + # state = state_postal[0] + # else: + # postal_code = state_postal[0] + state_postal[1] + # state = '' + # else: + # if len(state_postal[0]) == 6: + # postal_code = state_postal[0] + # state = '' + # else: + # postal_code = '' + # state = state_postal[0] + # else: + # city = '' + # state = '' + # postal_code = '' + # # state = state_postal[0] + # # postal_code = state_postal[1] + state_postal[2] + # country = address_parts[len(address_parts) - 1] + # if email_div and email_div.string: + # email = email_div.string + # else: + # email = '' + # if phone_div and phone_div.string: + # phone = phone_div.string + # else: + # phone = '' + # if website_div and website_div.string: + # website = website_div.string + # else: + # website = '' + + _logger.info(f"Processing {odoo_partner['name']} with email {odoo_partner['email']} and phone {odoo_partner['phone']} and website {odoo_partner['website']}") + + exist_odoo_name = self.env['res.partner'].search([('odoo_name', '=', odoo_partner['odoo_name'])]) + if not exist_odoo_name: + exist_email = self.env['res.partner'].search([('email', '=', odoo_partner['email'])]) + if not exist_email: + exist_phone = self.env['res.partner'].search([('phone', '=', odoo_partner['phone'])]) + if not exist_phone: + exist_website = self.env['res.partner'].search([('website', '=', odoo_partner['website'])]) + + if not exist_website or odoo_partner['website'] == '': + self.env['res.partner'].create(odoo_partner) + # self.env['res.partner'].create({ + # 'name': partner_name, + # 'street': street_address, + # 'city': city, + # 'state_id': state_id.id if state_id else False, + # 'zip': postal_code, + # 'country_id': country_id.id if country_id else False, + # 'email': email, + # 'phone': phone, + # 'website': website, + # 'is_odoo_partner': True, + # }) \ No newline at end of file diff --git a/bemade_odoo_partner_scrapper/static/src/js/odoo_scrapper.js b/bemade_odoo_partner_scrapper/static/src/js/odoo_scrapper.js new file mode 100644 index 0000000..f3b0ba6 --- /dev/null +++ b/bemade_odoo_partner_scrapper/static/src/js/odoo_scrapper.js @@ -0,0 +1,71 @@ +/** @odoo-module **/ + +import ListController from 'web.ListController'; +import ListView from 'web.ListView'; +import KanbanController from 'web.KanbanController'; +import KanbanView from 'web.KanbanView'; +const viewRegistry = require('web.view_registry'); + +const OdooScrapperListController = ListController.extend({ + // buttons_template must match the t-name on the template for the button (static xml) + buttons_template: 'odoo_scrapper.list_view_buttons', + events: _.extend({}, ListController.prototype.events, { + 'click .o_button_get_partner': '_onGetPartnerClick', + }), + // This may need to be async function() if there needs to be an await this._rpc({ ... }); call to not reload early + _onGetPartnerClick: function () { + this._rpc({ + model: 'res.partner', + method: 'get_odoo_partner', + args: [], + }).then(() => { + this.reload(); + }); + // Couldn't test this for real, but it runs the action. May need a this.reload() + }, +}); + +const OdooScrapperListView = ListView.extend({ + config: _.extend({}, ListView.prototype.config, { + Controller: OdooScrapperListController, + }), +}); + +// key must match with the js_class attribute of the tree view you want to modify +viewRegistry.add('res_partner_odoo_scrapper_tree', OdooScrapperListView); + + + + + + +const OdooScrapperKanbanController = KanbanController.extend({ + // buttons_template must match the t-name on the template for the button (static xml) + buttons_template: 'odoo_scrapper.kanban_view_buttons', + events: _.extend({}, KanbanController.prototype.events, { + 'click .o_button_get_partner': '_onGetPartnerClick', + }), + // This may need to be async function() if there needs to be an await this._rpc({ ... }); call to not reload early + _onGetPartnerClick: function () { + this._rpc({ + model: 'res.partner', + method: 'get_odoo_partner', + args: [], + }).then(() => { + this.reload(); + }); + // Couldn't test this for real, but it runs the action. May need a this.reload() + }, +}); + +const OdooScrapperKanbanView = KanbanView.extend({ + config: _.extend({}, KanbanView.prototype.config, { + Controller: OdooScrapperKanbanController, + }), +}); + +// key must match with the js_class attribute of the tree view you want to modify +viewRegistry.add('res_partner_odoo_scrapper_kanban', OdooScrapperKanbanView); + + + diff --git a/bemade_odoo_partner_scrapper/static/src/xml/odoo_scrapper_templates.xml b/bemade_odoo_partner_scrapper/static/src/xml/odoo_scrapper_templates.xml new file mode 100644 index 0000000..942044b --- /dev/null +++ b/bemade_odoo_partner_scrapper/static/src/xml/odoo_scrapper_templates.xml @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/bemade_odoo_partner_scrapper/views/res_partner_views.xml b/bemade_odoo_partner_scrapper/views/res_partner_views.xml new file mode 100644 index 0000000..1dd2f6e --- /dev/null +++ b/bemade_odoo_partner_scrapper/views/res_partner_views.xml @@ -0,0 +1,41 @@ + + + + + res.partner.tree.inherit + res.partner + + + + res_partner_odoo_scrapper_tree + + + + + + + + + res.partner.kanban.inherit + res.partner + + + + res_partner_odoo_scrapper_kanban + + + + + + res.partner.odoo.search + res.partner + + + + + + + + + +