odoo scrapper

bemade_odoo_partner_scrapper/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+
+from . import models

bemade_odoo_partner_scrapper/__manifest__.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+{
+    'name': 'Bemade Odoo Partner Scrapper',
+    'version': '1.0.0',
+    'category': 'Administration',
+    'summary': 'Module for scraping partners from odoo.com.',
+    'description': """
+    This module enables the scraping of partners from odoo.com. It allows for the collection and management of partner contact information.
+
+    Main Features:
+    * Automatically scrape partners from odoo.com.
+    * Extracts partner name, website, and contact information.
+    * Manage contact information of partners.
+    """,
+    'sequence': 10,
+    'license': 'GPL-3',
+    'author': 'Bemade',
+    'website': 'https://www.bemade.org',
+    'depends': [],
+    'data': [
+        # No security
+        'views/res_partner_views.xml',
+    ],
+    "assets": {
+        "web.assets_backend": [
+            "bemade_odoo_partner_scrapper/static/src/js/odoo_scrapper.js",
+        ],
+        "web.assets_qweb": [
+            "bemade_odoo_partner_scrapper/static/src/xml/odoo_scrapper_templates.xml",
+        ],
+    },
+    'demo': [],
+    'installable': True,
+    'application': False,
+    'auto_install': False
+}

bemade_odoo_partner_scrapper/models/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+
+from . import res_partner

bemade_odoo_partner_scrapper/models/res_partner.py

@@ -0,0 +1,86 @@
+from odoo import models, fields, api, _
+import requests
+from bs4 import BeautifulSoup
+
+
+class ResPartner(models.Model):
+    _inherit = 'res.partner'
+
+    is_odoo_partner = fields.Boolean(string="Is Odoo Partner", default=False)
+    is_odoo_user = fields.Boolean(string="Is Odoo User", default=False)
+
+    @api.model
+    def get_odoo_partner(self):
+        # Load the webpage at the given url
+        page = requests.get("https://www.odoo.com/fr_FR/partners/country/canada-36")
+        # Parse the HTML content
+        soup = BeautifulSoup(page.content, 'html.parser')
+        div = soup.find(id="ref_content")
+        partners_url = set()
+        other_pages = set()
+
+        for link in div.find_all('a'):
+            href = link.get('href')
+            if href is not None:
+                # Ignore any URL containing 'country/canada-36'
+                if '/page/' in href:
+                    # Add to set. Duplicates will be ignored automatically.
+                    other_pages.add(href)
+
+        for link in div.find_all('a'):
+            href = link.get('href')
+            if href is not None and href != '':
+                # Ignore any URL containing 'country/canada-36'
+                if 'country/canada-36' not in href:
+                    # Add to set. Duplicates will be ignored automatically.
+                    partners_url.add(href.split('#', 1)[0].split('?', 1)[0])
+
+        for other_page in other_pages:
+            page = requests.get("https://www.odoo.com" + other_page)
+            soup = BeautifulSoup(page.content, 'html.parser')
+            div = soup.find(id="ref_content")
+            for link in div.find_all('a'):
+                href = link.get('href')
+                if href is not None and href != '':
+                    # Ignore any URL containing 'country/canada-36'
+                    if 'country/canada-36' not in href:
+                        # Add to set. Duplicates will be ignored automatically.
+                        partners_url.add(href.split('#', 1)[0].split('?', 1)[0])
+
+        for partner_url in partners_url:
+            page = requests.get("https://www.odoo.com" + partner_url)
+            soup = BeautifulSoup(page.content, 'html.parser')
+            partner_name = soup.find(id="partner_name")
+            address_div = soup.find('span', itemprop='streetAddress')
+            email_div = soup.find('span', itemprop='email')
+            phone_div = soup.find('span', itemprop='telephone')
+            website_div = soup.find('span', itemprop='website')
+            if address_div is not None:
+                # Extract address components
+                address_parts = address_div.get_text(separator="\n").split("\n")
+                street_address = address_parts[0]
+                city_state_postal = address_parts[len(address_parts)-2].split(', ')
+                city = city_state_postal[0]
+                state_postal = city_state_postal[1].split(' ')
+                state = state_postal[0]
+                postal_code = state_postal[1] + state_postal[2]
+                country = address_parts[len(address_parts)-1]
+            if email_div and email_div.string:
+                email = email_div.string
+            if phone_div and phone_div.string:
+                phone = phone_div.string
+            if website_div and website_div.string:
+                website = website_div.string
+
+            # self.env['res.partner'].create({
+            #     'name': partner_name,
+            #     'street': street_address,
+            #     'city': city,
+            #     'state_id': state,
+            #     'zip': postal_code,
+            #     'country_id': country,
+            #     'email': email,
+            #     'phone': phone,
+            #     'website': website,
+            #     'is_odoo_partner': True,
+            # })

bemade_odoo_partner_scrapper/static/src/js/odoo_scrapper.js

@@ -0,0 +1,34 @@
+/** @odoo-module **/
+
+import ListController from 'web.ListController';
+import ListView from 'web.ListView';
+const viewRegistry = require('web.view_registry');
+
+const OdooScrapperListController = ListController.extend({
+    // buttons_template must match the t-name on the template for the button (static xml)
+    buttons_template: 'odoo_scrapper.list_view_buttons',
+    events: _.extend({}, ListController.prototype.events, {
+        'click .o_button_get_partner': '_onGetPartnerClick',
+    }),
+    // This may need to be async function() if there needs to be an await this._rpc({ ... }); call to not reload early
+    _onGetPartnerClick: function () {
+        this._rpc({
+            model: 'res.partner',
+            method: 'get_odoo_partner',
+            args: [],
+        }).then(() => {
+            this.reload();
+        });
+        // Couldn't test this for real, but it runs the action. May need a this.reload()
+    },
+});
+
+const OdooScrapperListView = ListView.extend({
+    config: _.extend({}, ListView.prototype.config, {
+        Controller: OdooScrapperListController,
+    }),
+});
+
+// key must match with the js_class attribute of the tree view you want to modify
+viewRegistry.add('res_partner_odoo_scrapper_tree', OdooScrapperListView);
+

bemade_odoo_partner_scrapper/static/src/xml/odoo_scrapper_templates.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<templates>
+
+    <t t-extend="ListView.buttons" t-name="odoo_scrapper.list_view_buttons"> <!-- t-name must match with js controller button template -->
+        <t t-jquery="button.o_list_button_add" t-operation="after">
+            <button type="button"
+                    class="btn btn-primary ml4 o_button_get_partner"
+                    title="Sync Aliases"> Get Odoo Partner </button>
+        </t>
+    </t>
+
+</templates>

bemade_odoo_partner_scrapper/views/res_partner_views.xml

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8"?>
+<odoo>
+
+    <record id="view_partner_tree_inherit" model="ir.ui.view">
+        <field name="name">res.partner.tree.inherit</field>
+        <field name="model">res.partner</field>
+        <field name="inherit_id" ref="base.view_partner_tree"/>
+        <field name="arch" type="xml">
+            <xpath expr="//tree" position="attributes">
+                <attribute name="js_class">res_partner_odoo_scrapper_tree</attribute>
+            </xpath>
+            <xpath expr="//tree" position="inside">
+                <field name="is_odoo_partner"/>
+            </xpath>
+        </field>
+    </record>
+
+</odoo>