diff --git a/msg_attachments_to_mail_message/models/ir_attachment.py b/msg_attachments_to_mail_message/models/ir_attachment.py index 4f8fd41..ac0349a 100644 --- a/msg_attachments_to_mail_message/models/ir_attachment.py +++ b/msg_attachments_to_mail_message/models/ir_attachment.py @@ -23,6 +23,10 @@ import psycopg2 import os import shutil from odoo.tools import config +import time +import random +import socket +import tempfile _logger = logging.getLogger(__name__) @@ -37,6 +41,9 @@ _msg_import_logger.setLevel(logging.ERROR) class IrAttachment(models.Model): _inherit = "ir.attachment" + # Cache au niveau de la classe + _msg_conversion_cache = {} + msg_processed = fields.Boolean(string="MSG Processed", default=False) mail_message_id = fields.Many2one("mail.message", string="Created Mail Message") @@ -97,6 +104,106 @@ class IrAttachment(models.Model): # Replace any combination of whitespace (including newlines) with a single space return " ".join(str(value).split()) + def _detect_encoding(self, data): + """ + Detect the encoding of the data. + + Args: + data (bytes): Data to analyze + + Returns: + str: Detected encoding (e.g., 'utf-8', 'iso-8859-1', etc.) + """ + try: + import chardet + result = chardet.detect(data) + encoding = result['encoding'] if result and result['encoding'] else 'utf-8' + _logger.debug("Detected encoding: %s with confidence: %s", + encoding, result.get('confidence', 0)) + return encoding + except ImportError: + _logger.info("chardet not installed, defaulting to utf-8") + return 'utf-8' + except Exception as e: + _logger.warning("Error detecting encoding: %s", str(e)) + return 'utf-8' + + def _retry_operation(self, operation, max_retries=3, delay=1): + """ + Retry an operation with exponential backoff. + + Args: + operation (callable): Function to retry + max_retries (int): Maximum number of retry attempts + delay (int): Initial delay between retries in seconds + + Returns: + Any: Result of the operation + """ + last_error = None + for attempt in range(max_retries): + try: + return operation() + except Exception as e: + last_error = e + if attempt < max_retries - 1: + sleep_time = delay * (2 ** attempt) # Exponential backoff + _logger.warning( + "Operation failed (attempt %d/%d): %s. Retrying in %d seconds...", + attempt + 1, max_retries, str(e), sleep_time + ) + time.sleep(sleep_time) + else: + _logger.error( + "Operation failed after %d attempts: %s", + max_retries, str(e) + ) + raise last_error + + def _get_cache_key(self, data): + """ + Generate a cache key for the data. + + Args: + data (bytes): Data to generate key for + + Returns: + str: Cache key + """ + import hashlib + return hashlib.sha256(data).hexdigest() + + @api.model + def _get_conversion_cache(self): + """ + Get the conversion cache. + + Returns: + dict: Conversion cache + """ + return self._msg_conversion_cache + + def _cached_msg_to_eml(self, msg_data): + """ + Cached version of MSG to EML conversion. + + Args: + msg_data (bytes): MSG file data + + Returns: + str: EML file data + """ + cache = self._get_conversion_cache() + cache_key = self._get_cache_key(msg_data) + + if cache_key in cache: + _logger.debug("Using cached conversion result") + return cache[cache_key] + + result = self._msg_to_eml(msg_data) + cache[cache_key] = result + return result + def _msg_to_eml(self, msg_data): """ Convert MSG file data to EML format. @@ -203,7 +310,26 @@ class IrAttachment(models.Model): for att in msg_file.attachments: try: filename = att.longFilename or att.shortFilename - if filename: + if not filename: + _logger.info("Skipping attachment with no filename") + continue + + _logger.info("=== Processing attachment: %s ===", filename) + _logger.info("Attachment type: %s", type(att.data).__name__) + _logger.info("Attachment size: %d bytes", len(att.data) if hasattr(att.data, '__len__') else -1) + + # Check if attachment is an email file (MSG or EML) + if isinstance(att.data, extract_msg.msg_classes.message.Message): + _logger.info("Processing as MSG file: %s", filename) + _logger.info("MSG attributes: %s", dir(att.data)) + mime_type = "application/vnd.ms-outlook" + elif filename.lower().endswith('.eml'): + _logger.info("Processing as EML file: %s", filename) + mime_type = "message/rfc822" + else: + # Regular attachment + _logger.info("Processing as regular attachment: %s", filename) + _logger.info("Content type: %s", getattr(att, 'content_type', 'unknown')) part = MIMEBase("application", "octet-stream") part.set_payload(att.data) encoders.encode_base64(part) @@ -212,17 +338,89 @@ class IrAttachment(models.Model): f"attachment; filename={filename}", ) email_msg.attach(part) + _logger.info("Regular attachment processed successfully") + continue + + # Process email file (MSG or EML) + _logger.info("Creating MIME part with type: %s", mime_type) + maintype, subtype = mime_type.split('/') + part = MIMEBase(maintype, subtype) + + # Get the raw data + if isinstance(att.data, extract_msg.msg_classes.message.Message): + _logger.info("Extracting raw bytes from MSG") + try: + # Try to get raw data using asEmailMessage + _logger.info("Converting MSG to email message") + email_message = att.data.asEmailMessage() + raw_data = email_message.as_bytes() + _logger.info("Successfully converted MSG to email message: %d bytes", len(raw_data)) + except Exception as e: + _logger.warning("Failed to convert MSG to email message: %s", str(e)) + try: + # Try to export as bytes + _logger.info("Trying to export MSG as bytes") + raw_data = att.data.exportBytes() + _logger.info("Successfully exported MSG as bytes: %d bytes", len(raw_data)) + except Exception as e: + _logger.warning("Failed to export MSG as bytes: %s", str(e)) + # Last resort: try to get the raw data directly + if hasattr(att.data, '_raw'): + _logger.info("Using _raw attribute") + raw_data = att.data._raw + else: + _logger.warning("Could not get raw data from MSG, skipping attachment") + continue + else: + _logger.info("Using raw data directly for EML") + raw_data = att.data + + if raw_data: + _logger.info("Raw data size: %d bytes", len(raw_data) if raw_data else 0) + + # Check if the data is base64 encoded + try: + # Try to decode base64 if it looks like base64 + if isinstance(raw_data, str) and all(c in 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=' for c in raw_data): + _logger.info("Data appears to be base64 encoded, decoding...") + import base64 + raw_data = base64.b64decode(raw_data) + _logger.info("Successfully decoded base64 data: %d bytes", len(raw_data)) + except Exception as e: + _logger.warning("Failed to decode base64 data: %s", str(e)) + + # Set the payload and encode + part.set_payload(raw_data) + _logger.info("Encoding attachment in base64") + encoders.encode_base64(part) + + # Add headers + _logger.info("Adding headers to MIME part") + part.add_header( + "Content-Disposition", + f"attachment; filename={filename}", + ) + part.add_header( + "Content-Type", + mime_type, + name=filename + ) + + _logger.info("Attaching processed email file to message") + email_msg.attach(part) + _logger.info("=== Finished processing %s ===", filename) + else: + _logger.warning("No raw data available for %s, skipping", filename) + except Exception as e: _logger.warning( "Failed to process attachment %s: %s - Attachment data type: %s, Content: %s", filename, str(e), type(att.data), - str(att.data)[:100] if att.data else "None" + str(att.data)[:100] if att.data else "None", + exc_info=True ) - # Move problematic attachment to review directory - self._move_to_review_dir('msg_conversion') - return email_msg.as_string() except Exception as e: @@ -318,17 +516,21 @@ class IrAttachment(models.Model): _logger.debug("MSG file already processed: %s", self.name) return True - try: - # Convert base64 to bytes with validation + def convert_msg(): msg_data = base64.b64decode(self.datas) if not isinstance(msg_data, bytes): raise ValueError(f"Expected bytes after b64decode, got {type(msg_data)}") - eml_content = self._msg_to_eml(msg_data) + # Use cached conversion + eml_content = self._cached_msg_to_eml(msg_data) if not isinstance(eml_content, str): raise ValueError(f"Expected string EML content, got {type(eml_content)}") + return eml_content - except (ValueError, TypeError) as e: + try: + # Use retry mechanism for conversion + eml_content = self._retry_operation(convert_msg) + except Exception as e: _logger.error("Data conversion error for %s: %s", self.name, str(e)) return False @@ -521,3 +723,35 @@ class IrAttachment(models.Model): _logger.error("[%s] Error processing attachment: %s", attachment.name or 'Unknown', str(e)) return res + + @api.model + def process_msg_attachments_batch(self, attachment_ids, batch_size=10): + """ + Process multiple MSG attachments in batches. + + Args: + attachment_ids (list): List of attachment IDs to process + batch_size (int): Number of attachments to process in each batch + + Returns: + dict: Processing results + """ + results = {'success': [], 'failed': []} + attachments = self.browse(attachment_ids) + + for i in range(0, len(attachments), batch_size): + batch = attachments[i:i + batch_size] + for attachment in batch: + try: + if attachment.process_msg_as_email(): + results['success'].append(attachment.id) + else: + results['failed'].append(attachment.id) + except Exception as e: + _logger.error( + "Error processing attachment %s: %s", + attachment.name, str(e) + ) + results['failed'].append(attachment.id) + + return results diff --git a/msg_attachments_to_mail_message/requirements.txt b/msg_attachments_to_mail_message/requirements.txt new file mode 100644 index 0000000..82f63e6 --- /dev/null +++ b/msg_attachments_to_mail_message/requirements.txt @@ -0,0 +1,3 @@ +chardet +extract-msg +PyPDF2