diff --git a/bemade_odoo_partner_scrapper/__manifest__.py b/bemade_odoo_partner_scrapper/__manifest__.py
index 9e389e0..a1eb2c5 100644
--- a/bemade_odoo_partner_scrapper/__manifest__.py
+++ b/bemade_odoo_partner_scrapper/__manifest__.py
@@ -25,8 +25,6 @@
"assets": {
"web.assets_backend": [
"bemade_odoo_partner_scrapper/static/src/js/odoo_scrapper.js",
- ],
- "web.assets_qweb": [
"bemade_odoo_partner_scrapper/static/src/xml/odoo_scrapper_templates.xml",
],
},
diff --git a/bemade_odoo_partner_scrapper/models/res_partner.py b/bemade_odoo_partner_scrapper/models/res_partner.py
index 8c283f3..e478def 100644
--- a/bemade_odoo_partner_scrapper/models/res_partner.py
+++ b/bemade_odoo_partner_scrapper/models/res_partner.py
@@ -2,6 +2,8 @@ from odoo import models, fields, api, _
import requests
from bs4 import BeautifulSoup
import logging
+from base64 import b64encode
+import re
_logger = logging.getLogger(__name__)
@@ -11,13 +13,59 @@ class ResPartner(models.Model):
is_odoo_partner = fields.Boolean(string="Is Odoo Partner", default=False, tracking=True)
is_odoo_user = fields.Boolean(string="Is Odoo User", default=False, tracking=True)
- odoo_name = fields.Char(string="Name on Odoo", tracking=True)
- odoo_id = fields.Char(string="ID on Odoo", tracking=True)
+ odoo_name = fields.Char(string="Name on Odoo")
+ odoo_note = fields.Html(string="Note on Odoo")
+ odoo_id = fields.Char(string="ID on Odoo")
odoo_url = fields.Char(string="Odoo Partner URL", tracking=True, index=True)
+ odoo_partner_type = fields.Selection(
+ [('learning', 'Learning'),
+ ('ready', 'Ready'),
+ ('silver', 'Silver'),
+ ('gold', 'Gold')],
+ string="Partner Type",
+ tracking=True)
+
+ def set_image_from_url(self, url):
+ response = requests.get(url)
+ if response.status_code == 200:
+ self.image_1920 = b64encode(response.content).decode('utf-8')
+ else:
+ raise ValueError('Could not download image: got response code {}'.format(response.status_code))
@api.model
def get_odoo_partner(self):
+ def extract_client_data(url):
+ page = requests.get("https://www.odoo.com" + url)
+ soup = BeautifulSoup(page.content, 'html.parser')
+ odoo_name = soup.find(id='partner_name').text
+ odoo_note = soup.find('div', class_='col-lg-8 mt32')
+
+ main_image = soup.find('img', {'class': 'img img-fluid d-block mx-auto mb16'})
+ image_link = main_image['src']
+
+ client = self.env['res.partner'].search([('odoo_name','=',odoo_name)])
+
+ if not client:
+ client = client.create({
+ 'name':odoo_name,
+ 'is_company': True,
+ 'odoo_name':odoo_name,
+ 'odoo_note':odoo_note,
+ 'is_odoo_user': True
+ })
+ client.set_image_from_url(image_link)
+
+ return(client)
+
+ def process_partner_relation(partner, client):
+ print (f"{partner.name} Odoo Partner of {client.name}")
+ self.env['res.partner.relation'].create({
+ 'left_partner_id' : partner.id,
+ 'right_partner_id' : client.id,
+ 'type_id' : self.env.ref('bemade_odoo_partner_scrapper.rel_type_odoo_partner').id
+ })
+
def extract_partner_data(url):
page = requests.get("https://www.odoo.com" + url)
soup = BeautifulSoup(page.content, 'html.parser')
@@ -25,12 +73,31 @@ class ResPartner(models.Model):
partner_data = {}
partner_data['name'] = soup.find(id="partner_name").text
+
+ partner_type = soup.find('h3', class_='col-lg-12 text-center text-muted')
+ if partner_type and partner_type.span:
+ partner_data['odoo_partner_type'] = partner_type.span.text.strip().lower()
+
partner_data['odoo_name'] = soup.find(id="partner_name").text
address_div = soup.find('span', itemprop='streetAddress')
email_div = soup.find('span', itemprop='email')
phone_div = soup.find('span', itemprop='telephone')
website_div = soup.find('span', itemprop='website')
+ main_image = soup.find('img', {'class': 'img img-fluid d-block mx-auto mb16'})
+ image_link = main_image['src']
+
+ clients_url = set()
+ for link in soup.find_all('a'):
+ href = link.get('href')
+ if href is not None:
+ # Ignore any URL containing 'country/canada-36'
+ if '/customers/' in href:
+ # Add to set. Duplicates will be ignored automatically.
+ clients_url.add(href)
+
+ state_data = None
+
if address_div is not None:
# Extract address components
address_parts = address_div.get_text(separator="\n").split("\n")
@@ -61,14 +128,20 @@ class ResPartner(models.Model):
# partner_data['country'] = address_parts[len(address_parts) - 1]
- partner_data['email'] = email_div.string if email_div and email_div.string else ''
- partner_data['phone'] = phone_div.string if phone_div and phone_div.string else ''
- partner_data['website'] = website_div.string if website_div and website_div.string else ''
+ partner_data['email'] = email_div.string if email_div and email_div.string else False
+ partner_data['phone'] = phone_div.string if phone_div and phone_div.string else False
+ partner_data['website'] = website_div.string if website_div and website_div.string else False
partner_data['is_odoo_partner'] = True
partner_data['is_odoo_user'] = True
+ partner_data['is_company'] = True
+ partner_data['image_link'] = image_link
+ partner_data['clients_url'] = clients_url
if state_data:
- partner_data['state_id'] = self.env['res.country.state'].search([('code', '=', state_data)]).id
+ country_id = self.env['res.country'].search([('code', '=', 'CA')]).id
+ partner_data['state_id'] = self.env['res.country.state'].search([('code', '=', state_data),('country_id', '=', country_id)]).id
+
+ partner_data = {k: v for k, v in partner_data.items() if v is not False}
return partner_data
@@ -109,89 +182,41 @@ class ResPartner(models.Model):
partners_url.add(href.split('#', 1)[0].split('?', 1)[0])
for partner_url in partners_url:
- page = requests.get("https://www.odoo.com" + partner_url)
- soup = BeautifulSoup(page.content, 'html.parser')
+ # page = requests.get("https://www.odoo.com" + partner_url)
+ # soup = BeautifulSoup(page.content, 'html.parser')
odoo_partner = extract_partner_data(partner_url)
- print (odoo_partner)
+ _logger.info(f"Processing {odoo_partner['name']}")
- # partner_name = soup.find(id="partner_name").text
- # address_div = soup.find('span', itemprop='streetAddress')
- # email_div = soup.find('span', itemprop='email')
- # phone_div = soup.find('span', itemprop='telephone')
- # website_div = soup.find('span', itemprop='website')
- # if address_div is not None:
- # # Extract address components
- # address_parts = address_div.get_text(separator="\n").split("\n")
- # street_address = address_parts[0]
- # city_state_postal = address_parts[len(address_parts) - 2].split(', ')
- #
- # if city_state_postal != [',']:
- # city = city_state_postal[0]
- # state_postal = city_state_postal[1].split(' ')
- #
- # if len(state_postal) == 3 and len(state_postal[2]) == 3:
- # postal_code = state_postal[1] + state_postal[2]
- # state = state_postal[0]
- # elif len(state_postal) == 2 and state_postal[1]:
- # if len(state_postal[1]) == 6:
- # postal_code = state_postal[1]
- # state = state_postal[0]
- # if len(state_postal[1]) == 3:
- # if len(state_postal[0]) == 2:
- # postal_code = state_postal[1]
- # state = state_postal[0]
- # else:
- # postal_code = state_postal[0] + state_postal[1]
- # state = ''
- # else:
- # if len(state_postal[0]) == 6:
- # postal_code = state_postal[0]
- # state = ''
- # else:
- # postal_code = ''
- # state = state_postal[0]
- # else:
- # city = ''
- # state = ''
- # postal_code = ''
- # # state = state_postal[0]
- # # postal_code = state_postal[1] + state_postal[2]
- # country = address_parts[len(address_parts) - 1]
- # if email_div and email_div.string:
- # email = email_div.string
- # else:
- # email = ''
- # if phone_div and phone_div.string:
- # phone = phone_div.string
- # else:
- # phone = ''
- # if website_div and website_div.string:
- # website = website_div.string
- # else:
- # website = ''
+ image_url = False
+ if odoo_partner['image_link']:
+ image_url = odoo_partner['image_link']
+ del odoo_partner['image_link']
- _logger.info(f"Processing {odoo_partner['name']} with email {odoo_partner['email']} and phone {odoo_partner['phone']} and website {odoo_partner['website']}")
+ print(odoo_partner['clients_url'])
+ if odoo_partner['clients_url']:
+ clients_url = odoo_partner['clients_url']
+ del odoo_partner['clients_url']
+ else:
+ clients_url = set()
+ print (clients_url)
exist_odoo_name = self.env['res.partner'].search([('odoo_name', '=', odoo_partner['odoo_name'])])
- if not exist_odoo_name:
- exist_email = self.env['res.partner'].search([('email', '=', odoo_partner['email'])])
- if not exist_email:
- exist_phone = self.env['res.partner'].search([('phone', '=', odoo_partner['phone'])])
- if not exist_phone:
- exist_website = self.env['res.partner'].search([('website', '=', odoo_partner['website'])])
+ if exist_odoo_name:
+ new_partner = exist_odoo_name
+ else:
+ exist_email = self.env['res.partner'].search([('email', '=', odoo_partner['email'])]) if 'email' in odoo_partner else False
+ if exist_email:
+ new_partner = exist_email
+ else:
+ exist_phone = self.env['res.partner'].search([('phone', '=', odoo_partner['phone'])]) if 'phone' in odoo_partner else False
+ if exist_phone:
+ new_partner = exist_phone
+ else:
+ new_partner = self.env['res.partner'].create(odoo_partner)
+ if image_url:
+ new_partner.set_image_from_url(image_url)
- if not exist_website or odoo_partner['website'] == '':
- self.env['res.partner'].create(odoo_partner)
- # self.env['res.partner'].create({
- # 'name': partner_name,
- # 'street': street_address,
- # 'city': city,
- # 'state_id': state_id.id if state_id else False,
- # 'zip': postal_code,
- # 'country_id': country_id.id if country_id else False,
- # 'email': email,
- # 'phone': phone,
- # 'website': website,
- # 'is_odoo_partner': True,
- # })
\ No newline at end of file
+ for client_url in clients_url:
+ client = extract_client_data(client_url.get('href'))
+ process_partner_relation(new_partner, client)
diff --git a/bemade_odoo_partner_scrapper/static/src/js/odoo_scrapper.js b/bemade_odoo_partner_scrapper/static/src/js/odoo_scrapper.js
index f3b0ba6..cd7b9ef 100644
--- a/bemade_odoo_partner_scrapper/static/src/js/odoo_scrapper.js
+++ b/bemade_odoo_partner_scrapper/static/src/js/odoo_scrapper.js
@@ -2,8 +2,10 @@
import ListController from 'web.ListController';
import ListView from 'web.ListView';
+
import KanbanController from 'web.KanbanController';
import KanbanView from 'web.KanbanView';
+
const viewRegistry = require('web.view_registry');
const OdooScrapperListController = ListController.extend({
@@ -25,7 +27,7 @@ const OdooScrapperListController = ListController.extend({
},
});
-const OdooScrapperListView = ListView.extend({
+export const OdooScrapperListView = ListView.extend({
config: _.extend({}, ListView.prototype.config, {
Controller: OdooScrapperListController,
}),
@@ -34,11 +36,6 @@ const OdooScrapperListView = ListView.extend({
// key must match with the js_class attribute of the tree view you want to modify
viewRegistry.add('res_partner_odoo_scrapper_tree', OdooScrapperListView);
-
-
-
-
-
const OdooScrapperKanbanController = KanbanController.extend({
// buttons_template must match the t-name on the template for the button (static xml)
buttons_template: 'odoo_scrapper.kanban_view_buttons',
@@ -58,7 +55,7 @@ const OdooScrapperKanbanController = KanbanController.extend({
},
});
-const OdooScrapperKanbanView = KanbanView.extend({
+export const OdooScrapperKanbanView = KanbanView.extend({
config: _.extend({}, KanbanView.prototype.config, {
Controller: OdooScrapperKanbanController,
}),
diff --git a/bemade_odoo_partner_scrapper/static/src/xml/odoo_scrapper_templates.xml b/bemade_odoo_partner_scrapper/static/src/xml/odoo_scrapper_templates.xml
index 942044b..af0a1dd 100644
--- a/bemade_odoo_partner_scrapper/static/src/xml/odoo_scrapper_templates.xml
+++ b/bemade_odoo_partner_scrapper/static/src/xml/odoo_scrapper_templates.xml
@@ -9,16 +9,12 @@
-
-
-
+
+
-
-
-
-
-
+
+
\ No newline at end of file
diff --git a/bemade_odoo_partner_scrapper/views/res_partner_views.xml b/bemade_odoo_partner_scrapper/views/res_partner_views.xml
index 1dd2f6e..94d5ac2 100644
--- a/bemade_odoo_partner_scrapper/views/res_partner_views.xml
+++ b/bemade_odoo_partner_scrapper/views/res_partner_views.xml
@@ -1,6 +1,19 @@
+
+ res.partner.form.inherit
+ res.partner
+
+
+
+
+
+
+
+
+
+
res.partner.tree.inherit
res.partner
@@ -10,7 +23,9 @@
res_partner_odoo_scrapper_tree
+
+
@@ -23,6 +38,15 @@
res_partner_odoo_scrapper_kanban
+
+
+
+
+
+
+
+
+