From 540e7f17d3e573d2e241deee4a325e2a88044b3d Mon Sep 17 00:00:00 2001 From: Marc Durepos Date: Wed, 15 May 2024 21:35:14 -0400 Subject: [PATCH] bemade_odoo_partner_scrapper moved to bemade-tools --- bemade_odoo_partner_scrapper/__init__.py | 3 - bemade_odoo_partner_scrapper/__manifest__.py | 36 --- bemade_odoo_partner_scrapper/data/demo.xml | 106 ------- .../data/res_partner_relation_type.xml | 37 --- .../models/__init__.py | 3 - .../models/res_partner.py | 264 ------------------ .../static/src/js/odoo_scrapper.js | 68 ----- .../src/xml/odoo_scrapper_templates.xml | 20 -- .../views/res_partner_views.xml | 86 ------ 9 files changed, 623 deletions(-) delete mode 100644 bemade_odoo_partner_scrapper/__init__.py delete mode 100644 bemade_odoo_partner_scrapper/__manifest__.py delete mode 100644 bemade_odoo_partner_scrapper/data/demo.xml delete mode 100644 bemade_odoo_partner_scrapper/data/res_partner_relation_type.xml delete mode 100644 bemade_odoo_partner_scrapper/models/__init__.py delete mode 100644 bemade_odoo_partner_scrapper/models/res_partner.py delete mode 100644 bemade_odoo_partner_scrapper/static/src/js/odoo_scrapper.js delete mode 100644 bemade_odoo_partner_scrapper/static/src/xml/odoo_scrapper_templates.xml delete mode 100644 bemade_odoo_partner_scrapper/views/res_partner_views.xml diff --git a/bemade_odoo_partner_scrapper/__init__.py b/bemade_odoo_partner_scrapper/__init__.py deleted file mode 100644 index 5305644..0000000 --- a/bemade_odoo_partner_scrapper/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -*- coding: utf-8 -*- - -from . import models \ No newline at end of file diff --git a/bemade_odoo_partner_scrapper/__manifest__.py b/bemade_odoo_partner_scrapper/__manifest__.py deleted file mode 100644 index 4cbe3a7..0000000 --- a/bemade_odoo_partner_scrapper/__manifest__.py +++ /dev/null @@ -1,36 +0,0 @@ -# -*- coding: utf-8 -*- -{ - 'name': 'Bemade Odoo Partner Scrapper', - 'version': '17.0.0.0.0', - 'category': 'Administration', - 'summary': 'Module for scraping partners from odoo.com.', - 'description': """ - This module enables the scraping of partners from odoo.com. It allows for the collection and management of partner contact information. - - Main Features: - * Automatically scrape partners from odoo.com. - * Extracts partner name, website, and contact information. - * Manage contact information of partners. - """, - 'sequence': 10, - 'license': 'AGPL-3', - 'author': 'Bemade', - 'website': 'https://www.bemade.org', - 'depends': ['base', 'contacts', 'sale_management', 'sale' , 'partner_multi_relation'], - 'data': [ - # No security - 'views/res_partner_views.xml', - # Commented for migration to 17, raises error on update - 'data/res_partner_relation_type.xml' - ], - # "assets": { - # "web.assets_backend": [ - # "bemade_odoo_partner_scrapper/static/src/js/odoo_scrapper.js", - # "bemade_odoo_partner_scrapper/static/src/xml/odoo_scrapper_templates.xml", - # ], - # }, - 'demo': [], - 'installable': True, - 'application': False, - 'auto_install': False -} diff --git a/bemade_odoo_partner_scrapper/data/demo.xml b/bemade_odoo_partner_scrapper/data/demo.xml deleted file mode 100644 index c47c023..0000000 --- a/bemade_odoo_partner_scrapper/data/demo.xml +++ /dev/null @@ -1,106 +0,0 @@ - - - - - - Is assistant of - Has assistant - p - p - - - Is competitor of - Is competitor of - c - c - - - - Has worked for - Has former employee - p - c - - - - Washing Companies - - - Washing Gold - - - - Washing Silver - - - - Washing Services - - - - - Great Washing Powder Company - - 1 - Le Bourget du Lac - 73377 - +33 4 49 23 44 54 - - 93, Press Avenue - great@yourcompany.example.com - http://www.great.com - - - Best Washing Powder Company - - 1 - Champs sur Marne - 77420 - - best@yourcompany.example.com - +33 1 64 61 04 01 - 12 rue Albert Einstein - http://www.best.com/ - - - Super Washing Powder Company - - 1 - 3rd Floor, Room 3-C, - Carretera Panamericana, Km 1, Urb. Delgado Chalbaud - Caracas - 1090 - super@yourcompany.example.com - +58 212 681 0538 - - super.com - - - - - - - - - - - - - - - - - - diff --git a/bemade_odoo_partner_scrapper/data/res_partner_relation_type.xml b/bemade_odoo_partner_scrapper/data/res_partner_relation_type.xml deleted file mode 100644 index fe644ad..0000000 --- a/bemade_odoo_partner_scrapper/data/res_partner_relation_type.xml +++ /dev/null @@ -1,37 +0,0 @@ - - - - - - Is Odoo Partner Of - Is Odoo Client Of - - - - - - - - Odoo Partner - - - Learning Partner - - - - Ready Partner - - - - Silver Partner - - - - Gold Partner - - - diff --git a/bemade_odoo_partner_scrapper/models/__init__.py b/bemade_odoo_partner_scrapper/models/__init__.py deleted file mode 100644 index cd5f4d8..0000000 --- a/bemade_odoo_partner_scrapper/models/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -*- coding: utf-8 -*- - -from . import res_partner \ No newline at end of file diff --git a/bemade_odoo_partner_scrapper/models/res_partner.py b/bemade_odoo_partner_scrapper/models/res_partner.py deleted file mode 100644 index b991d0d..0000000 --- a/bemade_odoo_partner_scrapper/models/res_partner.py +++ /dev/null @@ -1,264 +0,0 @@ -from odoo import models, fields, api, _ -import requests -from bs4 import BeautifulSoup -import logging -from base64 import b64encode -import re - -_logger = logging.getLogger(__name__) - - -class ResPartner(models.Model): - _inherit = 'res.partner' - - is_odoo_partner = fields.Boolean(string="Is Odoo Partner", default=False, tracking=True) - is_odoo_user = fields.Boolean(string="Is Odoo User", default=False, tracking=True) - odoo_name = fields.Char(string="Name on Odoo") - odoo_note = fields.Html(string="Note on Odoo") - odoo_id = fields.Char(string="ID on Odoo") - odoo_url = fields.Char(string="Odoo Partner URL", tracking=True, index=True) - odoo_page = fields.Html(string="Odoo Partner/Referral Page") - odoo_page_update = fields.Date(string="Odoo Page Update") - odoo_partner_type = fields.Selection( - [('learning', 'Learning'), - ('ready', 'Ready'), - ('silver', 'Silver'), - ('gold', 'Gold')], - string="Partner Type", - tracking=True) - - @api.model - def change_color_on_kanban(self): - for record in self: - color = 0 - if record.odoo_partner_type == 'learning': - color = 2 - elif record.odoo_partner_type == 'ready': - color = 10 - elif record.odoo_partner_type == 'silver': - color = 7 - elif record.odoo_partner_type == 'gold': - color = 3 - else: - if record.is_odoo_user: - color = 4 - record.color = color - - color = fields.Integer('Color Index', compute="change_color_on_kanban") - - def set_image_from_url(self, url): - response = requests.get(url) - if response.status_code == 200: - self.image_1920 = b64encode(response.content).decode('utf-8') - _logger.info(f"Image for {self.name} set from {url}") - else: - raise ValueError('Could not download image: got response code {}'.format(response.status_code)) - - @api.model - def get_odoo_partner(self): - - def extract_client_data(soup): - odoo_name = soup.find(id='partner_name').text - odoo_note = soup.find('div', class_='col-lg-8 mt32') - - main_image = soup.find('img', {'class': 'img img-fluid d-block mx-auto mb16'}) - image_link = main_image['src'] - - exist_client = self.env['res.partner'].search([('odoo_name', '=', odoo_name)]) - - if not exist_client: - exist_client = exist_client.create({ - 'name': odoo_name, - 'is_company': True, - 'odoo_name':odoo_name, - 'odoo_note':odoo_note, - 'is_odoo_user': True, - 'odoo_page': soup, - }) - exist_client.set_image_from_url(image_link) - _logger.info(f"Client {exist_client.name} created") - - return exist_client - - def process_partner_relation(partner, client): - if partner.id != client.id: - self.env['res.partner.relation'].create({ - 'left_partner_id' : partner.id, - 'right_partner_id' : client.id, - 'type_id' : self.env.ref('bemade_odoo_partner_scrapper.rel_type_odoo_partner').id - }) - _logger.info(f"Partner {partner.name} Odoo Partner of {client.name}") - else: - _logger.info(f"Partner {partner.name} is the same as client {client.name}") - - def extract_partner_data(url): - page = requests.get("https://www.odoo.com" + url) - soup = BeautifulSoup(page.content, 'html.parser') - - partner_data = {} - - partner_data['name'] = soup.find(id="partner_name").text - partner_type = soup.find('h3', class_='col-lg-12 text-center text-muted') - - if partner_type and partner_type.span: - partner_data['odoo_partner_type'] = partner_type.span.text.strip().lower() - - partner_data['odoo_name'] = soup.find(id="partner_name").text - address_div = soup.find('span', itemprop='streetAddress') - email_div = soup.find('span', itemprop='email') - phone_div = soup.find('span', itemprop='telephone') - website_div = soup.find('span', itemprop='website') - - main_image = soup.find('img', {'class': 'img img-fluid d-block mx-auto mb16'}) - image_link = main_image['src'] - - clients_url = set() - for link in soup.find_all('a'): - href = link.get('href') - if href is not None: - # Ignore any URL containing 'country/canada-36' - if '/customers/' in href: - # Add to set. Duplicates will be ignored automatically. - clients_url.add(href) - - state_data = None - - if address_div is not None: - # Extract address components - address_parts = address_div.get_text(separator="\n").split("\n") - partner_data['street'] = address_parts[0] - city_state_postal = address_parts[len(address_parts) - 2].split(', ') - - if city_state_postal != [',']: - partner_data['city'] = city_state_postal[0] - state_postal = city_state_postal[1].split(' ') - if len(state_postal) == 3 and len(state_postal[2]) == 3: - partner_data['zip'] = state_postal[1] + state_postal[2] - state_data = state_postal[0] - elif len(state_postal) == 2 and state_postal[1]: - if len(state_postal[1]) == 6: - partner_data['zip'] = state_postal[1] - state_data = state_postal[0] - if len(state_postal[1]) == 3: - if len(state_postal[0]) == 2: - partner_data['zip'] = state_postal[1] - state_data = state_postal[0] - else: - partner_data['zip'] = state_postal[0] + state_postal[1] - else: - if len(state_postal[0]) == 6: - partner_data['zip'] = state_postal[0] - else: - state_data = state_postal[0] - - # partner_data['country'] = address_parts[len(address_parts) - 1] - - partner_data['email'] = email_div.string if email_div and email_div.string else False - partner_data['phone'] = phone_div.string if phone_div and phone_div.string else False - partner_data['website'] = website_div.string if website_div and website_div.string else False - partner_data['is_odoo_partner'] = True - partner_data['is_odoo_user'] = True - partner_data['is_company'] = True - partner_data['image_link'] = image_link - partner_data['clients_url'] = clients_url - - if state_data: - country_id = self.env['res.country'].search([('code', '=', 'CA')]).id - partner_data['state_id'] = self.env['res.country.state'].search([('code', '=', state_data),('country_id', '=', country_id)]).id - - partner_data = {k: v for k, v in partner_data.items() if v is not False} - - return partner_data - - # Load the webpage at the given url - page = requests.get("https://www.odoo.com/fr_FR/partners/country/canada-36") - # Parse the HTML content - soup = BeautifulSoup(page.content, 'html.parser') - div = soup.find(id="ref_content") - partners_url = set() - other_pages = set() - - for link in div.find_all('a'): - href = link.get('href') - if href is not None: - # Ignore any URL containing 'country/canada-36' - if '/page/' in href: - # Add to set. Duplicates will be ignored automatically. - other_pages.add(href) - - for link in div.find_all('a'): - href = link.get('href') - if href is not None and href != '': - # Ignore any URL containing 'country/canada-36' - if 'country/canada-36' not in href: - # Add to set. Duplicates will be ignored automatically. - partners_url.add(href.split('#', 1)[0].split('?', 1)[0]) - - for other_page in other_pages: - page = requests.get("https://www.odoo.com" + other_page) - soup = BeautifulSoup(page.content, 'html.parser') - div = soup.find(id="ref_content") - for link in div.find_all('a'): - href = link.get('href') - if href is not None and href != '': - # Ignore any URL containing 'country/canada-36' - if 'country/canada-36' not in href: - # Add to set. Duplicates will be ignored automatically. - partners_url.add(href.split('#', 1)[0].split('?', 1)[0]) - - for partner_url in partners_url: - # page = requests.get("https://www.odoo.com" + partner_url) - # soup = BeautifulSoup(page.content, 'html.parser') - - odoo_partner = extract_partner_data(partner_url) - _logger.info(f"Processing {odoo_partner['name']}") - - image_url = False - if odoo_partner['image_link']: - image_url = odoo_partner['image_link'] - del odoo_partner['image_link'] - _logger.info(f"Image for {odoo_partner['name']} set from {image_url}") - - if odoo_partner['clients_url']: - clients_url = odoo_partner['clients_url'] - else: - clients_url = set() - - if 'clients_url' in odoo_partner: - del odoo_partner['clients_url'] - - exist_odoo_name = self.env['res.partner'].search([('odoo_name', '=', odoo_partner['odoo_name'])]) - if exist_odoo_name: - _logger.info(f"Partner Odoo Name {odoo_partner['name']} already exist") - new_partner = exist_odoo_name - else: - exist_email = self.env['res.partner'].search([('email', '=', odoo_partner['email'])]) if 'email' in odoo_partner else False - if exist_email: - _logger.info(f"Partner Email {odoo_partner['name']} already exist") - new_partner = exist_email - else: - exist_phone = self.env['res.partner'].search([('phone', '=', odoo_partner['phone'])]) if 'phone' in odoo_partner else False - if exist_phone: - _logger.info(f"Partner Phone {odoo_partner['name']} already exist") - new_partner = exist_phone - else: - _logger.info(f"Partner {odoo_partner['name']} created") - new_partner = self.env['res.partner'].create(odoo_partner) - if image_url: - new_partner.set_image_from_url(image_url) - - for client_url in clients_url: - if 'http' not in client_url: - exist_url = self.env['res.partner'].search([('odoo_url', '=', client_url)]) - # add condition on odoo_date, actually not pulling data if date exist - if exist_url and exist_url.odoo_page: - soup = exist_url.odoo_page - _logger.info(f"Client {exist_url.name} reloading soup from db") - else: - print("https://www.odoo.com" + client_url) - page = requests.get("https://www.odoo.com" + client_url) - soup = BeautifulSoup(page.content, 'html.parser') - _logger.info(f"Client {exist_url.name} read from odoo.com") - - client = extract_client_data(soup) - process_partner_relation(new_partner, client) diff --git a/bemade_odoo_partner_scrapper/static/src/js/odoo_scrapper.js b/bemade_odoo_partner_scrapper/static/src/js/odoo_scrapper.js deleted file mode 100644 index cd7b9ef..0000000 --- a/bemade_odoo_partner_scrapper/static/src/js/odoo_scrapper.js +++ /dev/null @@ -1,68 +0,0 @@ -/** @odoo-module **/ - -import ListController from 'web.ListController'; -import ListView from 'web.ListView'; - -import KanbanController from 'web.KanbanController'; -import KanbanView from 'web.KanbanView'; - -const viewRegistry = require('web.view_registry'); - -const OdooScrapperListController = ListController.extend({ - // buttons_template must match the t-name on the template for the button (static xml) - buttons_template: 'odoo_scrapper.list_view_buttons', - events: _.extend({}, ListController.prototype.events, { - 'click .o_button_get_partner': '_onGetPartnerClick', - }), - // This may need to be async function() if there needs to be an await this._rpc({ ... }); call to not reload early - _onGetPartnerClick: function () { - this._rpc({ - model: 'res.partner', - method: 'get_odoo_partner', - args: [], - }).then(() => { - this.reload(); - }); - // Couldn't test this for real, but it runs the action. May need a this.reload() - }, -}); - -export const OdooScrapperListView = ListView.extend({ - config: _.extend({}, ListView.prototype.config, { - Controller: OdooScrapperListController, - }), -}); - -// key must match with the js_class attribute of the tree view you want to modify -viewRegistry.add('res_partner_odoo_scrapper_tree', OdooScrapperListView); - -const OdooScrapperKanbanController = KanbanController.extend({ - // buttons_template must match the t-name on the template for the button (static xml) - buttons_template: 'odoo_scrapper.kanban_view_buttons', - events: _.extend({}, KanbanController.prototype.events, { - 'click .o_button_get_partner': '_onGetPartnerClick', - }), - // This may need to be async function() if there needs to be an await this._rpc({ ... }); call to not reload early - _onGetPartnerClick: function () { - this._rpc({ - model: 'res.partner', - method: 'get_odoo_partner', - args: [], - }).then(() => { - this.reload(); - }); - // Couldn't test this for real, but it runs the action. May need a this.reload() - }, -}); - -export const OdooScrapperKanbanView = KanbanView.extend({ - config: _.extend({}, KanbanView.prototype.config, { - Controller: OdooScrapperKanbanController, - }), -}); - -// key must match with the js_class attribute of the tree view you want to modify -viewRegistry.add('res_partner_odoo_scrapper_kanban', OdooScrapperKanbanView); - - - diff --git a/bemade_odoo_partner_scrapper/static/src/xml/odoo_scrapper_templates.xml b/bemade_odoo_partner_scrapper/static/src/xml/odoo_scrapper_templates.xml deleted file mode 100644 index af0a1dd..0000000 --- a/bemade_odoo_partner_scrapper/static/src/xml/odoo_scrapper_templates.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/bemade_odoo_partner_scrapper/views/res_partner_views.xml b/bemade_odoo_partner_scrapper/views/res_partner_views.xml deleted file mode 100644 index 22402a4..0000000 --- a/bemade_odoo_partner_scrapper/views/res_partner_views.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - - - res.partner.form.inherit - res.partner - - - -
- Odoo User - Odoo Partner -
-
-

Partner

-
- - - - - - - - - - - - - - - - - - - - res.partner.tree.inherit - res.partner - - - - res_partner_odoo_scrapper_tree - - - - - - - - - - - res.partner.kanban.inherit - res.partner - - - - res_partner_odoo_scrapper_kanban - - - - - - oe_kanban_global_click o_kanban_record_has_image_fill o_res_partner_kanban oe_kanban_color_#{record.color} - - - - - - - - - - - res.partner.odoo.search - res.partner - - - - - - - - - -