from odoo import models, fields, api, _ import requests from bs4 import BeautifulSoup import logging from base64 import b64encode import re _logger = logging.getLogger(__name__) class ResPartner(models.Model): _inherit = 'res.partner' is_odoo_partner = fields.Boolean(string="Is Odoo Partner", default=False, tracking=True) is_odoo_user = fields.Boolean(string="Is Odoo User", default=False, tracking=True) odoo_name = fields.Char(string="Name on Odoo") odoo_note = fields.Html(string="Note on Odoo") odoo_id = fields.Char(string="ID on Odoo") odoo_url = fields.Char(string="Odoo Partner URL", tracking=True, index=True) odoo_partner_type = fields.Selection( [('learning', 'Learning'), ('ready', 'Ready'), ('silver', 'Silver'), ('gold', 'Gold')], string="Partner Type", tracking=True) def set_image_from_url(self, url): response = requests.get(url) if response.status_code == 200: self.image_1920 = b64encode(response.content).decode('utf-8') else: raise ValueError('Could not download image: got response code {}'.format(response.status_code)) @api.model def get_odoo_partner(self): def extract_client_data(url): page = requests.get("https://www.odoo.com" + url) soup = BeautifulSoup(page.content, 'html.parser') odoo_name = soup.find(id='partner_name').text odoo_note = soup.find('div', class_='col-lg-8 mt32') main_image = soup.find('img', {'class': 'img img-fluid d-block mx-auto mb16'}) image_link = main_image['src'] client = self.env['res.partner'].search([('odoo_name','=',odoo_name)]) if not client: client = client.create({ 'name':odoo_name, 'is_company': True, 'odoo_name':odoo_name, 'odoo_note':odoo_note, 'is_odoo_user': True }) client.set_image_from_url(image_link) return(client) def process_partner_relation(partner, client): print (f"{partner.name} Odoo Partner of {client.name}") self.env['res.partner.relation'].create({ 'left_partner_id' : partner.id, 'right_partner_id' : client.id, 'type_id' : self.env.ref('bemade_odoo_partner_scrapper.rel_type_odoo_partner').id }) def extract_partner_data(url): page = requests.get("https://www.odoo.com" + url) soup = BeautifulSoup(page.content, 'html.parser') partner_data = {} partner_data['name'] = soup.find(id="partner_name").text partner_type = soup.find('h3', class_='col-lg-12 text-center text-muted') if partner_type and partner_type.span: partner_data['odoo_partner_type'] = partner_type.span.text.strip().lower() partner_data['odoo_name'] = soup.find(id="partner_name").text address_div = soup.find('span', itemprop='streetAddress') email_div = soup.find('span', itemprop='email') phone_div = soup.find('span', itemprop='telephone') website_div = soup.find('span', itemprop='website') main_image = soup.find('img', {'class': 'img img-fluid d-block mx-auto mb16'}) image_link = main_image['src'] clients_url = set() for link in soup.find_all('a'): href = link.get('href') if href is not None: # Ignore any URL containing 'country/canada-36' if '/customers/' in href: # Add to set. Duplicates will be ignored automatically. clients_url.add(href) state_data = None if address_div is not None: # Extract address components address_parts = address_div.get_text(separator="\n").split("\n") partner_data['street'] = address_parts[0] city_state_postal = address_parts[len(address_parts) - 2].split(', ') if city_state_postal != [',']: partner_data['city'] = city_state_postal[0] state_postal = city_state_postal[1].split(' ') if len(state_postal) == 3 and len(state_postal[2]) == 3: partner_data['zip'] = state_postal[1] + state_postal[2] state_data = state_postal[0] elif len(state_postal) == 2 and state_postal[1]: if len(state_postal[1]) == 6: partner_data['zip'] = state_postal[1] state_data = state_postal[0] if len(state_postal[1]) == 3: if len(state_postal[0]) == 2: partner_data['zip'] = state_postal[1] state_data = state_postal[0] else: partner_data['zip'] = state_postal[0] + state_postal[1] else: if len(state_postal[0]) == 6: partner_data['zip'] = state_postal[0] else: state_data = state_postal[0] # partner_data['country'] = address_parts[len(address_parts) - 1] partner_data['email'] = email_div.string if email_div and email_div.string else False partner_data['phone'] = phone_div.string if phone_div and phone_div.string else False partner_data['website'] = website_div.string if website_div and website_div.string else False partner_data['is_odoo_partner'] = True partner_data['is_odoo_user'] = True partner_data['is_company'] = True partner_data['image_link'] = image_link partner_data['clients_url'] = clients_url if state_data: country_id = self.env['res.country'].search([('code', '=', 'CA')]).id partner_data['state_id'] = self.env['res.country.state'].search([('code', '=', state_data),('country_id', '=', country_id)]).id partner_data = {k: v for k, v in partner_data.items() if v is not False} return partner_data # Load the webpage at the given url page = requests.get("https://www.odoo.com/fr_FR/partners/country/canada-36") # Parse the HTML content soup = BeautifulSoup(page.content, 'html.parser') div = soup.find(id="ref_content") partners_url = set() other_pages = set() for link in div.find_all('a'): href = link.get('href') if href is not None: # Ignore any URL containing 'country/canada-36' if '/page/' in href: # Add to set. Duplicates will be ignored automatically. other_pages.add(href) for link in div.find_all('a'): href = link.get('href') if href is not None and href != '': # Ignore any URL containing 'country/canada-36' if 'country/canada-36' not in href: # Add to set. Duplicates will be ignored automatically. partners_url.add(href.split('#', 1)[0].split('?', 1)[0]) for other_page in other_pages: page = requests.get("https://www.odoo.com" + other_page) soup = BeautifulSoup(page.content, 'html.parser') div = soup.find(id="ref_content") for link in div.find_all('a'): href = link.get('href') if href is not None and href != '': # Ignore any URL containing 'country/canada-36' if 'country/canada-36' not in href: # Add to set. Duplicates will be ignored automatically. partners_url.add(href.split('#', 1)[0].split('?', 1)[0]) for partner_url in partners_url: # page = requests.get("https://www.odoo.com" + partner_url) # soup = BeautifulSoup(page.content, 'html.parser') odoo_partner = extract_partner_data(partner_url) _logger.info(f"Processing {odoo_partner['name']}") image_url = False if odoo_partner['image_link']: image_url = odoo_partner['image_link'] del odoo_partner['image_link'] print(odoo_partner['clients_url']) if odoo_partner['clients_url']: clients_url = odoo_partner['clients_url'] del odoo_partner['clients_url'] else: clients_url = set() print (clients_url) exist_odoo_name = self.env['res.partner'].search([('odoo_name', '=', odoo_partner['odoo_name'])]) if exist_odoo_name: new_partner = exist_odoo_name else: exist_email = self.env['res.partner'].search([('email', '=', odoo_partner['email'])]) if 'email' in odoo_partner else False if exist_email: new_partner = exist_email else: exist_phone = self.env['res.partner'].search([('phone', '=', odoo_partner['phone'])]) if 'phone' in odoo_partner else False if exist_phone: new_partner = exist_phone else: new_partner = self.env['res.partner'].create(odoo_partner) if image_url: new_partner.set_image_from_url(image_url) for client_url in clients_url: client = extract_client_data(client_url.get('href')) process_partner_relation(new_partner, client)