Revert removal of partner scraper modules and fix caldav_sync spam.
This commit is contained in:
Marc Durepos
commit
c5062b6b2b
12 changed files with 674 additions and 4 deletions
3
bemade_odoo_partner_scrapper/__init__.py
Normal file
3
bemade_odoo_partner_scrapper/__init__.py
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
from . import models
|
||||||
36
bemade_odoo_partner_scrapper/__manifest__.py
Normal file
36
bemade_odoo_partner_scrapper/__manifest__.py
Normal file
|
|
@ -0,0 +1,36 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
{
|
||||||
|
'name': 'Bemade Odoo Partner Scrapper',
|
||||||
|
'version': '17.0.0.0.0',
|
||||||
|
'category': 'Administration',
|
||||||
|
'summary': 'Module for scraping partners from odoo.com.',
|
||||||
|
'description': """
|
||||||
|
This module enables the scraping of partners from odoo.com. It allows for the collection and management of partner contact information.
|
||||||
|
|
||||||
|
Main Features:
|
||||||
|
* Automatically scrape partners from odoo.com.
|
||||||
|
* Extracts partner name, website, and contact information.
|
||||||
|
* Manage contact information of partners.
|
||||||
|
""",
|
||||||
|
'sequence': 10,
|
||||||
|
'license': 'AGPL-3',
|
||||||
|
'author': 'Bemade',
|
||||||
|
'website': 'https://www.bemade.org',
|
||||||
|
'depends': ['base', 'contacts', 'sale_management', 'sale' , 'partner_multi_relation'],
|
||||||
|
'data': [
|
||||||
|
# No security
|
||||||
|
'views/res_partner_views.xml',
|
||||||
|
# Commented for migration to 17, raises error on update
|
||||||
|
'data/res_partner_relation_type.xml'
|
||||||
|
],
|
||||||
|
# "assets": {
|
||||||
|
# "web.assets_backend": [
|
||||||
|
# "bemade_odoo_partner_scrapper/static/src/js/odoo_scrapper.js",
|
||||||
|
# "bemade_odoo_partner_scrapper/static/src/xml/odoo_scrapper_templates.xml",
|
||||||
|
# ],
|
||||||
|
# },
|
||||||
|
'demo': [],
|
||||||
|
'installable': True,
|
||||||
|
'application': False,
|
||||||
|
'auto_install': False
|
||||||
|
}
|
||||||
106
bemade_odoo_partner_scrapper/data/demo.xml
Normal file
106
bemade_odoo_partner_scrapper/data/demo.xml
Normal file
|
|
@ -0,0 +1,106 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8" ?>
|
||||||
|
<odoo>
|
||||||
|
<!-- Added partner categories and partners to this file, because it
|
||||||
|
turned out to be a bad idea to rely on demo data in base module,
|
||||||
|
that can change from release to release. Only dependency on
|
||||||
|
countries remain. They are less likely to change/disappear.
|
||||||
|
-->
|
||||||
|
<!-- Partner relation types -->
|
||||||
|
<record id="rel_type_assistant" model="res.partner.relation.type">
|
||||||
|
<field name="name">Is assistant of</field>
|
||||||
|
<field name="name_inverse">Has assistant</field>
|
||||||
|
<field name="contact_type_left">p</field>
|
||||||
|
<field name="contact_type_right">p</field>
|
||||||
|
</record>
|
||||||
|
<record id="rel_type_competitor" model="res.partner.relation.type">
|
||||||
|
<field name="name">Is competitor of</field>
|
||||||
|
<field name="name_inverse">Is competitor of</field>
|
||||||
|
<field name="contact_type_left">c</field>
|
||||||
|
<field name="contact_type_right">c</field>
|
||||||
|
<field name="is_symmetric" eval="True" />
|
||||||
|
</record>
|
||||||
|
<record id="rel_type_has_worked_for" model="res.partner.relation.type">
|
||||||
|
<field name="name">Has worked for</field>
|
||||||
|
<field name="name_inverse">Has former employee</field>
|
||||||
|
<field name="contact_type_left">p</field>
|
||||||
|
<field name="contact_type_right">c</field>
|
||||||
|
</record>
|
||||||
|
<!-- Categories -->
|
||||||
|
<record id="res_partner_category_pmr_0" model="res.partner.category">
|
||||||
|
<field name="name">Washing Companies</field>
|
||||||
|
</record>
|
||||||
|
<record id="res_partner_category_pmr_4" model="res.partner.category">
|
||||||
|
<field name="name">Washing Gold</field>
|
||||||
|
<field name="parent_id" ref="res_partner_category_pmr_0" />
|
||||||
|
</record>
|
||||||
|
<record id="res_partner_category_pmr_5" model="res.partner.category">
|
||||||
|
<field name="name">Washing Silver</field>
|
||||||
|
<field name="parent_id" ref="res_partner_category_pmr_0" />
|
||||||
|
</record>
|
||||||
|
<record id="res_partner_category_pmr_11" model="res.partner.category">
|
||||||
|
<field name="name">Washing Services</field>
|
||||||
|
<field name="parent_id" ref="res_partner_category_pmr_0" />
|
||||||
|
</record>
|
||||||
|
<!-- Partners -->
|
||||||
|
<record id="res_partner_pmr_great" model="res.partner">
|
||||||
|
<field name="name">Great Washing Powder Company</field>
|
||||||
|
<field
|
||||||
|
name="category_id"
|
||||||
|
eval="[(6, 0, [ref('res_partner_category_pmr_11'), ref('res_partner_category_pmr_4')])]"
|
||||||
|
/>
|
||||||
|
<field name="is_company">1</field>
|
||||||
|
<field name="city">Le Bourget du Lac</field>
|
||||||
|
<field name="zip">73377</field>
|
||||||
|
<field name="phone">+33 4 49 23 44 54</field>
|
||||||
|
<field name="country_id" ref="base.fr" />
|
||||||
|
<field name="street">93, Press Avenue</field>
|
||||||
|
<field name="email">great@yourcompany.example.com</field>
|
||||||
|
<field name="website">http://www.great.com</field>
|
||||||
|
</record>
|
||||||
|
<record id="res_partner_pmr_best" model="res.partner">
|
||||||
|
<field name="name">Best Washing Powder Company</field>
|
||||||
|
<field
|
||||||
|
name="category_id"
|
||||||
|
eval="[(6, 0, [ref('res_partner_category_pmr_4'), ref('res_partner_category_pmr_11')])]"
|
||||||
|
/>
|
||||||
|
<field name="is_company">1</field>
|
||||||
|
<field name="city">Champs sur Marne</field>
|
||||||
|
<field name="zip">77420</field>
|
||||||
|
<field name="country_id" ref="base.fr" />
|
||||||
|
<field name="email">best@yourcompany.example.com</field>
|
||||||
|
<field name="phone">+33 1 64 61 04 01</field>
|
||||||
|
<field name="street">12 rue Albert Einstein</field>
|
||||||
|
<field name="website">http://www.best.com/</field>
|
||||||
|
</record>
|
||||||
|
<record id="res_partner_pmr_super" model="res.partner">
|
||||||
|
<field name="name">Super Washing Powder Company</field>
|
||||||
|
<field name="category_id" eval="[(6,0,[ref('res_partner_category_pmr_5')])]" />
|
||||||
|
<field name="is_company">1</field>
|
||||||
|
<field name="street">3rd Floor, Room 3-C,</field>
|
||||||
|
<field
|
||||||
|
name="street2"
|
||||||
|
>Carretera Panamericana, Km 1, Urb. Delgado Chalbaud</field>
|
||||||
|
<field name="city">Caracas</field>
|
||||||
|
<field name="zip">1090</field>
|
||||||
|
<field name="email">super@yourcompany.example.com</field>
|
||||||
|
<field name="phone">+58 212 681 0538</field>
|
||||||
|
<field name="country_id" ref="base.ve" />
|
||||||
|
<field name="website">super.com</field>
|
||||||
|
</record>
|
||||||
|
<!-- Relations -->
|
||||||
|
<record id="rel_1" model="res.partner.relation">
|
||||||
|
<field name="left_partner_id" ref="res_partner_pmr_great" />
|
||||||
|
<field name="right_partner_id" ref="res_partner_pmr_super" />
|
||||||
|
<field name="type_id" ref="rel_type_competitor" />
|
||||||
|
</record>
|
||||||
|
<record id="rel_2" model="res.partner.relation">
|
||||||
|
<field name="left_partner_id" ref="res_partner_pmr_best" />
|
||||||
|
<field name="right_partner_id" ref="res_partner_pmr_super" />
|
||||||
|
<field name="type_id" ref="rel_type_competitor" />
|
||||||
|
</record>
|
||||||
|
<record id="rel_3" model="res.partner.relation">
|
||||||
|
<field name="left_partner_id" ref="res_partner_pmr_great" />
|
||||||
|
<field name="right_partner_id" ref="res_partner_pmr_best" />
|
||||||
|
<field name="type_id" ref="rel_type_competitor" />
|
||||||
|
</record>
|
||||||
|
</odoo>
|
||||||
|
|
@ -0,0 +1,37 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8" ?>
|
||||||
|
<odoo>
|
||||||
|
<!-- Added partner categories and partners to this file, because it
|
||||||
|
turned out to be a bad idea to rely on demo data in base module,
|
||||||
|
that can change from release to release. Only dependency on
|
||||||
|
countries remain. They are less likely to change/disappear.
|
||||||
|
-->
|
||||||
|
<!-- Partner relation types -->
|
||||||
|
<record id="rel_type_odoo_partner" model="res.partner.relation.type">
|
||||||
|
<field name="name">Is Odoo Partner Of</field>
|
||||||
|
<field name="name_inverse">Is Odoo Client Of</field>
|
||||||
|
<!-- Commented two lines below due to data integrity issue when migrating to 17.0. Are they really needed? -->
|
||||||
|
<!-- <field name="contact_type_left">c</field>-->
|
||||||
|
<!-- <field name="contact_type_right">c</field>-->
|
||||||
|
</record>
|
||||||
|
|
||||||
|
<!-- Categories -->
|
||||||
|
<record id="res_partner_category_odoo_partner" model="res.partner.category">
|
||||||
|
<field name="name">Odoo Partner</field>
|
||||||
|
</record>
|
||||||
|
<record id="res_partner_category_odoo_partner_learning" model="res.partner.category">
|
||||||
|
<field name="name">Learning Partner</field>
|
||||||
|
<field name="parent_id" ref="res_partner_category_odoo_partner" />
|
||||||
|
</record>
|
||||||
|
<record id="res_partner_category_odoo_partner_ready" model="res.partner.category">
|
||||||
|
<field name="name">Ready Partner</field>
|
||||||
|
<field name="parent_id" ref="res_partner_category_odoo_partner" />
|
||||||
|
</record>
|
||||||
|
<record id="res_partner_category_odoo_partner_silver" model="res.partner.category">
|
||||||
|
<field name="name">Silver Partner</field>
|
||||||
|
<field name="parent_id" ref="res_partner_category_odoo_partner" />
|
||||||
|
</record>
|
||||||
|
<record id="res_partner_category_odoo_partner_gold" model="res.partner.category">
|
||||||
|
<field name="name">Gold Partner</field>
|
||||||
|
<field name="parent_id" ref="res_partner_category_odoo_partner" />
|
||||||
|
</record>
|
||||||
|
</odoo>
|
||||||
3
bemade_odoo_partner_scrapper/models/__init__.py
Normal file
3
bemade_odoo_partner_scrapper/models/__init__.py
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
from . import res_partner
|
||||||
264
bemade_odoo_partner_scrapper/models/res_partner.py
Normal file
264
bemade_odoo_partner_scrapper/models/res_partner.py
Normal file
|
|
@ -0,0 +1,264 @@
|
||||||
|
from odoo import models, fields, api, _
|
||||||
|
import requests
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
import logging
|
||||||
|
from base64 import b64encode
|
||||||
|
import re
|
||||||
|
|
||||||
|
_logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class ResPartner(models.Model):
|
||||||
|
_inherit = 'res.partner'
|
||||||
|
|
||||||
|
is_odoo_partner = fields.Boolean(string="Is Odoo Partner", default=False, tracking=True)
|
||||||
|
is_odoo_user = fields.Boolean(string="Is Odoo User", default=False, tracking=True)
|
||||||
|
odoo_name = fields.Char(string="Name on Odoo")
|
||||||
|
odoo_note = fields.Html(string="Note on Odoo")
|
||||||
|
odoo_id = fields.Char(string="ID on Odoo")
|
||||||
|
odoo_url = fields.Char(string="Odoo Partner URL", tracking=True, index=True)
|
||||||
|
odoo_page = fields.Html(string="Odoo Partner/Referral Page")
|
||||||
|
odoo_page_update = fields.Date(string="Odoo Page Update")
|
||||||
|
odoo_partner_type = fields.Selection(
|
||||||
|
[('learning', 'Learning'),
|
||||||
|
('ready', 'Ready'),
|
||||||
|
('silver', 'Silver'),
|
||||||
|
('gold', 'Gold')],
|
||||||
|
string="Partner Type",
|
||||||
|
tracking=True)
|
||||||
|
|
||||||
|
@api.model
|
||||||
|
def change_color_on_kanban(self):
|
||||||
|
for record in self:
|
||||||
|
color = 0
|
||||||
|
if record.odoo_partner_type == 'learning':
|
||||||
|
color = 2
|
||||||
|
elif record.odoo_partner_type == 'ready':
|
||||||
|
color = 10
|
||||||
|
elif record.odoo_partner_type == 'silver':
|
||||||
|
color = 7
|
||||||
|
elif record.odoo_partner_type == 'gold':
|
||||||
|
color = 3
|
||||||
|
else:
|
||||||
|
if record.is_odoo_user:
|
||||||
|
color = 4
|
||||||
|
record.color = color
|
||||||
|
|
||||||
|
color = fields.Integer('Color Index', compute="change_color_on_kanban")
|
||||||
|
|
||||||
|
def set_image_from_url(self, url):
|
||||||
|
response = requests.get(url)
|
||||||
|
if response.status_code == 200:
|
||||||
|
self.image_1920 = b64encode(response.content).decode('utf-8')
|
||||||
|
_logger.info(f"Image for {self.name} set from {url}")
|
||||||
|
else:
|
||||||
|
raise ValueError('Could not download image: got response code {}'.format(response.status_code))
|
||||||
|
|
||||||
|
@api.model
|
||||||
|
def get_odoo_partner(self):
|
||||||
|
|
||||||
|
def extract_client_data(soup):
|
||||||
|
odoo_name = soup.find(id='partner_name').text
|
||||||
|
odoo_note = soup.find('div', class_='col-lg-8 mt32')
|
||||||
|
|
||||||
|
main_image = soup.find('img', {'class': 'img img-fluid d-block mx-auto mb16'})
|
||||||
|
image_link = main_image['src']
|
||||||
|
|
||||||
|
exist_client = self.env['res.partner'].search([('odoo_name', '=', odoo_name)])
|
||||||
|
|
||||||
|
if not exist_client:
|
||||||
|
exist_client = exist_client.create({
|
||||||
|
'name': odoo_name,
|
||||||
|
'is_company': True,
|
||||||
|
'odoo_name':odoo_name,
|
||||||
|
'odoo_note':odoo_note,
|
||||||
|
'is_odoo_user': True,
|
||||||
|
'odoo_page': soup,
|
||||||
|
})
|
||||||
|
exist_client.set_image_from_url(image_link)
|
||||||
|
_logger.info(f"Client {exist_client.name} created")
|
||||||
|
|
||||||
|
return exist_client
|
||||||
|
|
||||||
|
def process_partner_relation(partner, client):
|
||||||
|
if partner.id != client.id:
|
||||||
|
self.env['res.partner.relation'].create({
|
||||||
|
'left_partner_id' : partner.id,
|
||||||
|
'right_partner_id' : client.id,
|
||||||
|
'type_id' : self.env.ref('bemade_odoo_partner_scrapper.rel_type_odoo_partner').id
|
||||||
|
})
|
||||||
|
_logger.info(f"Partner {partner.name} Odoo Partner of {client.name}")
|
||||||
|
else:
|
||||||
|
_logger.info(f"Partner {partner.name} is the same as client {client.name}")
|
||||||
|
|
||||||
|
def extract_partner_data(url):
|
||||||
|
page = requests.get("https://www.odoo.com" + url)
|
||||||
|
soup = BeautifulSoup(page.content, 'html.parser')
|
||||||
|
|
||||||
|
partner_data = {}
|
||||||
|
|
||||||
|
partner_data['name'] = soup.find(id="partner_name").text
|
||||||
|
partner_type = soup.find('h3', class_='col-lg-12 text-center text-muted')
|
||||||
|
|
||||||
|
if partner_type and partner_type.span:
|
||||||
|
partner_data['odoo_partner_type'] = partner_type.span.text.strip().lower()
|
||||||
|
|
||||||
|
partner_data['odoo_name'] = soup.find(id="partner_name").text
|
||||||
|
address_div = soup.find('span', itemprop='streetAddress')
|
||||||
|
email_div = soup.find('span', itemprop='email')
|
||||||
|
phone_div = soup.find('span', itemprop='telephone')
|
||||||
|
website_div = soup.find('span', itemprop='website')
|
||||||
|
|
||||||
|
main_image = soup.find('img', {'class': 'img img-fluid d-block mx-auto mb16'})
|
||||||
|
image_link = main_image['src']
|
||||||
|
|
||||||
|
clients_url = set()
|
||||||
|
for link in soup.find_all('a'):
|
||||||
|
href = link.get('href')
|
||||||
|
if href is not None:
|
||||||
|
# Ignore any URL containing 'country/canada-36'
|
||||||
|
if '/customers/' in href:
|
||||||
|
# Add to set. Duplicates will be ignored automatically.
|
||||||
|
clients_url.add(href)
|
||||||
|
|
||||||
|
state_data = None
|
||||||
|
|
||||||
|
if address_div is not None:
|
||||||
|
# Extract address components
|
||||||
|
address_parts = address_div.get_text(separator="\n").split("\n")
|
||||||
|
partner_data['street'] = address_parts[0]
|
||||||
|
city_state_postal = address_parts[len(address_parts) - 2].split(', ')
|
||||||
|
|
||||||
|
if city_state_postal != [',']:
|
||||||
|
partner_data['city'] = city_state_postal[0]
|
||||||
|
state_postal = city_state_postal[1].split(' ')
|
||||||
|
if len(state_postal) == 3 and len(state_postal[2]) == 3:
|
||||||
|
partner_data['zip'] = state_postal[1] + state_postal[2]
|
||||||
|
state_data = state_postal[0]
|
||||||
|
elif len(state_postal) == 2 and state_postal[1]:
|
||||||
|
if len(state_postal[1]) == 6:
|
||||||
|
partner_data['zip'] = state_postal[1]
|
||||||
|
state_data = state_postal[0]
|
||||||
|
if len(state_postal[1]) == 3:
|
||||||
|
if len(state_postal[0]) == 2:
|
||||||
|
partner_data['zip'] = state_postal[1]
|
||||||
|
state_data = state_postal[0]
|
||||||
|
else:
|
||||||
|
partner_data['zip'] = state_postal[0] + state_postal[1]
|
||||||
|
else:
|
||||||
|
if len(state_postal[0]) == 6:
|
||||||
|
partner_data['zip'] = state_postal[0]
|
||||||
|
else:
|
||||||
|
state_data = state_postal[0]
|
||||||
|
|
||||||
|
# partner_data['country'] = address_parts[len(address_parts) - 1]
|
||||||
|
|
||||||
|
partner_data['email'] = email_div.string if email_div and email_div.string else False
|
||||||
|
partner_data['phone'] = phone_div.string if phone_div and phone_div.string else False
|
||||||
|
partner_data['website'] = website_div.string if website_div and website_div.string else False
|
||||||
|
partner_data['is_odoo_partner'] = True
|
||||||
|
partner_data['is_odoo_user'] = True
|
||||||
|
partner_data['is_company'] = True
|
||||||
|
partner_data['image_link'] = image_link
|
||||||
|
partner_data['clients_url'] = clients_url
|
||||||
|
|
||||||
|
if state_data:
|
||||||
|
country_id = self.env['res.country'].search([('code', '=', 'CA')]).id
|
||||||
|
partner_data['state_id'] = self.env['res.country.state'].search([('code', '=', state_data),('country_id', '=', country_id)]).id
|
||||||
|
|
||||||
|
partner_data = {k: v for k, v in partner_data.items() if v is not False}
|
||||||
|
|
||||||
|
return partner_data
|
||||||
|
|
||||||
|
# Load the webpage at the given url
|
||||||
|
page = requests.get("https://www.odoo.com/fr_FR/partners/country/canada-36")
|
||||||
|
# Parse the HTML content
|
||||||
|
soup = BeautifulSoup(page.content, 'html.parser')
|
||||||
|
div = soup.find(id="ref_content")
|
||||||
|
partners_url = set()
|
||||||
|
other_pages = set()
|
||||||
|
|
||||||
|
for link in div.find_all('a'):
|
||||||
|
href = link.get('href')
|
||||||
|
if href is not None:
|
||||||
|
# Ignore any URL containing 'country/canada-36'
|
||||||
|
if '/page/' in href:
|
||||||
|
# Add to set. Duplicates will be ignored automatically.
|
||||||
|
other_pages.add(href)
|
||||||
|
|
||||||
|
for link in div.find_all('a'):
|
||||||
|
href = link.get('href')
|
||||||
|
if href is not None and href != '':
|
||||||
|
# Ignore any URL containing 'country/canada-36'
|
||||||
|
if 'country/canada-36' not in href:
|
||||||
|
# Add to set. Duplicates will be ignored automatically.
|
||||||
|
partners_url.add(href.split('#', 1)[0].split('?', 1)[0])
|
||||||
|
|
||||||
|
for other_page in other_pages:
|
||||||
|
page = requests.get("https://www.odoo.com" + other_page)
|
||||||
|
soup = BeautifulSoup(page.content, 'html.parser')
|
||||||
|
div = soup.find(id="ref_content")
|
||||||
|
for link in div.find_all('a'):
|
||||||
|
href = link.get('href')
|
||||||
|
if href is not None and href != '':
|
||||||
|
# Ignore any URL containing 'country/canada-36'
|
||||||
|
if 'country/canada-36' not in href:
|
||||||
|
# Add to set. Duplicates will be ignored automatically.
|
||||||
|
partners_url.add(href.split('#', 1)[0].split('?', 1)[0])
|
||||||
|
|
||||||
|
for partner_url in partners_url:
|
||||||
|
# page = requests.get("https://www.odoo.com" + partner_url)
|
||||||
|
# soup = BeautifulSoup(page.content, 'html.parser')
|
||||||
|
|
||||||
|
odoo_partner = extract_partner_data(partner_url)
|
||||||
|
_logger.info(f"Processing {odoo_partner['name']}")
|
||||||
|
|
||||||
|
image_url = False
|
||||||
|
if odoo_partner['image_link']:
|
||||||
|
image_url = odoo_partner['image_link']
|
||||||
|
del odoo_partner['image_link']
|
||||||
|
_logger.info(f"Image for {odoo_partner['name']} set from {image_url}")
|
||||||
|
|
||||||
|
if odoo_partner['clients_url']:
|
||||||
|
clients_url = odoo_partner['clients_url']
|
||||||
|
else:
|
||||||
|
clients_url = set()
|
||||||
|
|
||||||
|
if 'clients_url' in odoo_partner:
|
||||||
|
del odoo_partner['clients_url']
|
||||||
|
|
||||||
|
exist_odoo_name = self.env['res.partner'].search([('odoo_name', '=', odoo_partner['odoo_name'])])
|
||||||
|
if exist_odoo_name:
|
||||||
|
_logger.info(f"Partner Odoo Name {odoo_partner['name']} already exist")
|
||||||
|
new_partner = exist_odoo_name
|
||||||
|
else:
|
||||||
|
exist_email = self.env['res.partner'].search([('email', '=', odoo_partner['email'])]) if 'email' in odoo_partner else False
|
||||||
|
if exist_email:
|
||||||
|
_logger.info(f"Partner Email {odoo_partner['name']} already exist")
|
||||||
|
new_partner = exist_email
|
||||||
|
else:
|
||||||
|
exist_phone = self.env['res.partner'].search([('phone', '=', odoo_partner['phone'])]) if 'phone' in odoo_partner else False
|
||||||
|
if exist_phone:
|
||||||
|
_logger.info(f"Partner Phone {odoo_partner['name']} already exist")
|
||||||
|
new_partner = exist_phone
|
||||||
|
else:
|
||||||
|
_logger.info(f"Partner {odoo_partner['name']} created")
|
||||||
|
new_partner = self.env['res.partner'].create(odoo_partner)
|
||||||
|
if image_url:
|
||||||
|
new_partner.set_image_from_url(image_url)
|
||||||
|
|
||||||
|
for client_url in clients_url:
|
||||||
|
if 'http' not in client_url:
|
||||||
|
exist_url = self.env['res.partner'].search([('odoo_url', '=', client_url)])
|
||||||
|
# add condition on odoo_date, actually not pulling data if date exist
|
||||||
|
if exist_url and exist_url.odoo_page:
|
||||||
|
soup = exist_url.odoo_page
|
||||||
|
_logger.info(f"Client {exist_url.name} reloading soup from db")
|
||||||
|
else:
|
||||||
|
print("https://www.odoo.com" + client_url)
|
||||||
|
page = requests.get("https://www.odoo.com" + client_url)
|
||||||
|
soup = BeautifulSoup(page.content, 'html.parser')
|
||||||
|
_logger.info(f"Client {exist_url.name} read from odoo.com")
|
||||||
|
|
||||||
|
client = extract_client_data(soup)
|
||||||
|
process_partner_relation(new_partner, client)
|
||||||
68
bemade_odoo_partner_scrapper/static/src/js/odoo_scrapper.js
Normal file
68
bemade_odoo_partner_scrapper/static/src/js/odoo_scrapper.js
Normal file
|
|
@ -0,0 +1,68 @@
|
||||||
|
/** @odoo-module **/
|
||||||
|
|
||||||
|
import ListController from 'web.ListController';
|
||||||
|
import ListView from 'web.ListView';
|
||||||
|
|
||||||
|
import KanbanController from 'web.KanbanController';
|
||||||
|
import KanbanView from 'web.KanbanView';
|
||||||
|
|
||||||
|
const viewRegistry = require('web.view_registry');
|
||||||
|
|
||||||
|
const OdooScrapperListController = ListController.extend({
|
||||||
|
// buttons_template must match the t-name on the template for the button (static xml)
|
||||||
|
buttons_template: 'odoo_scrapper.list_view_buttons',
|
||||||
|
events: _.extend({}, ListController.prototype.events, {
|
||||||
|
'click .o_button_get_partner': '_onGetPartnerClick',
|
||||||
|
}),
|
||||||
|
// This may need to be async function() if there needs to be an await this._rpc({ ... }); call to not reload early
|
||||||
|
_onGetPartnerClick: function () {
|
||||||
|
this._rpc({
|
||||||
|
model: 'res.partner',
|
||||||
|
method: 'get_odoo_partner',
|
||||||
|
args: [],
|
||||||
|
}).then(() => {
|
||||||
|
this.reload();
|
||||||
|
});
|
||||||
|
// Couldn't test this for real, but it runs the action. May need a this.reload()
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
export const OdooScrapperListView = ListView.extend({
|
||||||
|
config: _.extend({}, ListView.prototype.config, {
|
||||||
|
Controller: OdooScrapperListController,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
// key must match with the js_class attribute of the tree view you want to modify
|
||||||
|
viewRegistry.add('res_partner_odoo_scrapper_tree', OdooScrapperListView);
|
||||||
|
|
||||||
|
const OdooScrapperKanbanController = KanbanController.extend({
|
||||||
|
// buttons_template must match the t-name on the template for the button (static xml)
|
||||||
|
buttons_template: 'odoo_scrapper.kanban_view_buttons',
|
||||||
|
events: _.extend({}, KanbanController.prototype.events, {
|
||||||
|
'click .o_button_get_partner': '_onGetPartnerClick',
|
||||||
|
}),
|
||||||
|
// This may need to be async function() if there needs to be an await this._rpc({ ... }); call to not reload early
|
||||||
|
_onGetPartnerClick: function () {
|
||||||
|
this._rpc({
|
||||||
|
model: 'res.partner',
|
||||||
|
method: 'get_odoo_partner',
|
||||||
|
args: [],
|
||||||
|
}).then(() => {
|
||||||
|
this.reload();
|
||||||
|
});
|
||||||
|
// Couldn't test this for real, but it runs the action. May need a this.reload()
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
export const OdooScrapperKanbanView = KanbanView.extend({
|
||||||
|
config: _.extend({}, KanbanView.prototype.config, {
|
||||||
|
Controller: OdooScrapperKanbanController,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
// key must match with the js_class attribute of the tree view you want to modify
|
||||||
|
viewRegistry.add('res_partner_odoo_scrapper_kanban', OdooScrapperKanbanView);
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -0,0 +1,20 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8" ?>
|
||||||
|
<templates id="template" xml:space="preserve">
|
||||||
|
|
||||||
|
<t t-extend="ListView.buttons" t-name="odoo_scrapper.list_view_buttons"> <!-- t-name must match with js controller button template -->
|
||||||
|
<t t-jquery="button.o_list_button_add" t-operation="after">
|
||||||
|
<button type="button"
|
||||||
|
class="btn btn-primary ml4 o_button_get_partner"
|
||||||
|
title="Get Odoo Partner"> Get Odoo Partner </button>
|
||||||
|
</t>
|
||||||
|
</t>
|
||||||
|
|
||||||
|
<t t-extend="KanbanView.buttons" t-name="odoo_scrapper.kanban_view_buttons">
|
||||||
|
<t t-jquery="button" t-operation="after">
|
||||||
|
<button type="button"
|
||||||
|
class="btn btn-primary ml4 o_button_get_partner"
|
||||||
|
title="Get Odoo Partner"> Get Odoo Partner </button>
|
||||||
|
</t>
|
||||||
|
</t>
|
||||||
|
|
||||||
|
</templates>
|
||||||
86
bemade_odoo_partner_scrapper/views/res_partner_views.xml
Normal file
86
bemade_odoo_partner_scrapper/views/res_partner_views.xml
Normal file
|
|
@ -0,0 +1,86 @@
|
||||||
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
|
<odoo>
|
||||||
|
|
||||||
|
<record id="view_partner_form_inherit" model="ir.ui.view">
|
||||||
|
<field name="name">res.partner.form.inherit</field>
|
||||||
|
<field name="model">res.partner</field>
|
||||||
|
<field name="inherit_id" ref="base.view_partner_form"/>
|
||||||
|
<field name="arch" type="xml">
|
||||||
|
<field name="company_type" position="after">
|
||||||
|
<br/>
|
||||||
|
<field name="is_odoo_user"/>Odoo User
|
||||||
|
<field name="is_odoo_partner"/>Odoo Partner
|
||||||
|
<br/>
|
||||||
|
<div invisible="is_odoo_partner == False">
|
||||||
|
<h3><field name="odoo_partner_type"/> Partner</h3>
|
||||||
|
</div>
|
||||||
|
</field>
|
||||||
|
<group name="container_row_2" position="after">
|
||||||
|
<group name="container_row_3">
|
||||||
|
<field name="relation_all_ids">
|
||||||
|
<tree>
|
||||||
|
<field name="other_partner_id" required="True" options="{'no_create': True}"
|
||||||
|
string="Customer" domain="[['relation_all_ids.type_selection_id.id', '=', 1]]"/>
|
||||||
|
<field name="date_start"/>
|
||||||
|
<field name="date_end"/>
|
||||||
|
<field name="active" invisible="1"/>
|
||||||
|
<field name="active" invisible="1"/>
|
||||||
|
</tree>
|
||||||
|
</field>
|
||||||
|
</group>
|
||||||
|
</group>
|
||||||
|
</field>
|
||||||
|
|
||||||
|
</record>
|
||||||
|
|
||||||
|
<record id="view_partner_tree_inherit" model="ir.ui.view">
|
||||||
|
<field name="name">res.partner.tree.inherit</field>
|
||||||
|
<field name="model">res.partner</field>
|
||||||
|
<field name="inherit_id" ref="base.view_partner_tree"/>
|
||||||
|
<field name="arch" type="xml">
|
||||||
|
<xpath expr="//tree" position="attributes">
|
||||||
|
<attribute name="js_class">res_partner_odoo_scrapper_tree</attribute>
|
||||||
|
</xpath>
|
||||||
|
<xpath expr="//tree" position="inside">
|
||||||
|
<field name="is_odoo_user"/>
|
||||||
|
<field name="is_odoo_partner"/>
|
||||||
|
<field name="odoo_partner_type"/>
|
||||||
|
</xpath>
|
||||||
|
</field>
|
||||||
|
</record>
|
||||||
|
|
||||||
|
<record id="view_partner_kanban_inherit" model="ir.ui.view">
|
||||||
|
<field name="name">res.partner.kanban.inherit</field>
|
||||||
|
<field name="model">res.partner</field>
|
||||||
|
<field name="inherit_id" ref="base.res_partner_kanban_view"/>
|
||||||
|
<field name="arch" type="xml">
|
||||||
|
<xpath expr="//kanban" position="attributes">
|
||||||
|
<attribute name="js_class">res_partner_odoo_scrapper_kanban</attribute>
|
||||||
|
</xpath>
|
||||||
|
<xpath expr="//kanban" position="inside">
|
||||||
|
<field name="odoo_partner_type"/>
|
||||||
|
</xpath>
|
||||||
|
<xpath expr="//div[hasclass('oe_kanban_global_click') and hasclass('o_kanban_record_has_image_fill') and hasclass('o_res_partner_kanban')]" position="attributes">
|
||||||
|
<attribute name="t-attf-class">oe_kanban_global_click o_kanban_record_has_image_fill o_res_partner_kanban oe_kanban_color_#{record.color}</attribute>
|
||||||
|
</xpath>
|
||||||
|
<xpath expr="//strong[hasclass('o_kanban_record_title') and hasclass('oe_partner_heading')]" position="before">
|
||||||
|
<strong t-if="record.odoo_partner_type.raw_value" class="o_kanban_record_subtitle oe_partner_heading">
|
||||||
|
<field name="odoo_partner_type"/>
|
||||||
|
</strong>
|
||||||
|
</xpath>
|
||||||
|
</field>
|
||||||
|
</record>
|
||||||
|
|
||||||
|
<record model="ir.ui.view" id="view_res_partner_odoo_search">
|
||||||
|
<field name="name">res.partner.odoo.search</field>
|
||||||
|
<field name="model">res.partner</field>
|
||||||
|
<field name="inherit_id" ref="base.view_res_partner_filter"/>
|
||||||
|
<field name="arch" type="xml">
|
||||||
|
<filter name="type_company" position="after">
|
||||||
|
<filter domain="[('is_odoo_partner','=','True')]" string="Odoo Partners" name="res_partner_odoo_partner_filter"/>
|
||||||
|
<filter domain="[('is_odoo_user','=','True')]" string="Odoo Users" name="res_partner_odoo_user_filter"/>
|
||||||
|
</filter>
|
||||||
|
</field>
|
||||||
|
</record>
|
||||||
|
|
||||||
|
</odoo>
|
||||||
1
bemade_odoo_partner_scrapper_js_only/__init__.py
Normal file
1
bemade_odoo_partner_scrapper_js_only/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
33
bemade_odoo_partner_scrapper_js_only/__manifest__.py
Normal file
33
bemade_odoo_partner_scrapper_js_only/__manifest__.py
Normal file
|
|
@ -0,0 +1,33 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
{
|
||||||
|
'name': 'Bemade Odoo Partner Scrapper JS Only',
|
||||||
|
'version': '17.0.1.0.0',
|
||||||
|
'category': 'Administration',
|
||||||
|
'summary': 'Module for scraping partners from odoo.com.',
|
||||||
|
'description': """
|
||||||
|
This module enables the scraping of partners from odoo.com. It allows for the collection and management of partner contact information.
|
||||||
|
|
||||||
|
Main Features:
|
||||||
|
* Automatically scrape partners from odoo.com.
|
||||||
|
* Extracts partner name, website, and contact information.
|
||||||
|
* Manage contact information of partners.
|
||||||
|
""",
|
||||||
|
'license': 'GPL-3',
|
||||||
|
'author': 'Bemade',
|
||||||
|
'website': 'https://www.bemade.org',
|
||||||
|
'depends': [
|
||||||
|
'bemade_odoo_partner_scrapper'
|
||||||
|
],
|
||||||
|
'data': [
|
||||||
|
],
|
||||||
|
"assets": {
|
||||||
|
"web.assets_backend": [
|
||||||
|
"bemade_odoo_partner_scrapper/static/src/js/odoo_scrapper.js",
|
||||||
|
"bemade_odoo_partner_scrapper/static/src/xml/odoo_scrapper_templates.xml",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
'demo': [],
|
||||||
|
'installable': True,
|
||||||
|
'application': False,
|
||||||
|
'auto_install': False
|
||||||
|
}
|
||||||
|
|
@ -6,7 +6,6 @@ from datetime import datetime, timezone
|
||||||
from icalendar import Calendar, Event, vCalAddress, vText, vWeekday
|
from icalendar import Calendar, Event, vCalAddress, vText, vWeekday
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
import dateutil
|
|
||||||
import re
|
import re
|
||||||
from pytz import timezone, utc
|
from pytz import timezone, utc
|
||||||
|
|
||||||
|
|
@ -320,7 +319,6 @@ class CalendarEvent(models.Model):
|
||||||
end = component.decoded('dtend')
|
end = component.decoded('dtend')
|
||||||
if isinstance(end, datetime):
|
if isinstance(end, datetime):
|
||||||
end = end.astimezone(utc).replace(tzinfo=None)
|
end = end.astimezone(utc).replace(tzinfo=None)
|
||||||
|
|
||||||
values = {
|
values = {
|
||||||
'name': str(component.get('summary')),
|
'name': str(component.get('summary')),
|
||||||
'start': start,
|
'start': start,
|
||||||
|
|
@ -339,12 +337,27 @@ class CalendarEvent(models.Model):
|
||||||
self.with_context({'caldav_no_sync': True}).create(values)
|
self.with_context({'caldav_no_sync': True}).create(values)
|
||||||
else:
|
else:
|
||||||
_logger.info(f"Updating with vals: {values}")
|
_logger.info(f"Updating with vals: {values}")
|
||||||
if recurrency_vals and recurrency_vals.get('recurrency'):
|
changed_vals = {}
|
||||||
|
# Don't update partner_ids if no change
|
||||||
|
if attendee_ids - existing_instance.partner_ids:
|
||||||
|
changed_vals.update({
|
||||||
|
'partner_ids',values.pop('partner_ids'),
|
||||||
|
})
|
||||||
|
|
||||||
|
# Don't write values that haven't changed
|
||||||
|
for key, val in values.items():
|
||||||
|
if getattr(existing_instance, key) != val:
|
||||||
|
changed_vals.update({key: values.pop(key)})
|
||||||
|
if (recurrency_vals and recurrency_vals.get('recurrency')
|
||||||
|
and (not existing_instance.recurrency or not
|
||||||
|
existing_instance.follow_recurrence)):
|
||||||
existing_instance.write({
|
existing_instance.write({
|
||||||
'recurrency': True,
|
'recurrency': True,
|
||||||
'follow_recurrence': True,
|
'follow_recurrence': True,
|
||||||
})
|
})
|
||||||
existing_instance.with_context({'caldav_no_sync': True}).write(values)
|
existing_instance.with_context({
|
||||||
|
'caldav_no_sync': True,
|
||||||
|
}).write(changed_vals)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _extract_component_text(component, subcomponent_name):
|
def _extract_component_text(component, subcomponent_name):
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue