Fix long running command actions (#1205)

Reduces risk of timeouts in case Icinga takes its time to respond.
The timeout of 15 seconds introduced earlier has been kept.

Chunk sizes were chosen as follows:
- 50: Expected disk writes (comments)
- 250: Only used for process checkresult
- 500: Cheap calculations (object features, check scheduling)

fixes #1204

application/forms/Command/Object/AcknowledgeProblemForm.php

@@ -20,6 +20,8 @@ use ipl\Validator\CallbackValidator;
 use ipl\Web\FormDecorator\IcingaFormDecorator;
 use ipl\Web\Widget\Icon;
 use Iterator;
+use LimitIterator;
+use NoRewindIterator;
 use Traversable;
 
 use function ipl\Stdlib\iterable_value_first;
@@ -182,8 +184,17 @@ class AcknowledgeProblemForm extends CommandForm
             'submit',
             'btn_submit',
             [
-                'required'  => true,
-                'label'     => tp('Acknowledge problem', 'Acknowledge problems', count($this->getObjects()))
+                'required'              => true,
+                'label'                 => tp(
+                    'Acknowledge problem',
+                    'Acknowledge problems',
+                    count($this->getObjects())
+                ),
+                'data-progress-label'   => tp(
+                    'Acknowledging problem',
+                    'Acknowledging problems',
+                    count($this->getObjects())
+                )
             ]
         );
 
@@ -196,22 +207,22 @@ class AcknowledgeProblemForm extends CommandForm
             return $this->isGrantedOn('icingadb/command/acknowledge-problem', $object);
         });
 
+        $command = new AcknowledgeProblemCommand();
+        $command->setComment($this->getValue('comment'));
+        $command->setAuthor($this->getAuth()->getUser()->getUsername());
+        $command->setNotify($this->getElement('notify')->isChecked());
+        $command->setSticky($this->getElement('sticky')->isChecked());
+        $command->setPersistent($this->getElement('persistent')->isChecked());
+
+        if (($expireTime = $this->getValue('expire_time')) !== null) {
+            /** @var DateTime $expireTime */
+            $command->setExpireTime($expireTime->getTimestamp());
+        }
+
         $granted->rewind(); // Forwards the pointer to the first element
-        if ($granted->valid()) {
-            $command = new AcknowledgeProblemCommand();
-            $command->setObjects($granted);
-            $command->setComment($this->getValue('comment'));
-            $command->setAuthor($this->getAuth()->getUser()->getUsername());
-            $command->setNotify($this->getElement('notify')->isChecked());
-            $command->setSticky($this->getElement('sticky')->isChecked());
-            $command->setPersistent($this->getElement('persistent')->isChecked());
-
-            if (($expireTime = $this->getValue('expire_time')) !== null) {
-                /** @var DateTime $expireTime */
-                $command->setExpireTime($expireTime->getTimestamp());
-            }
-
-            yield $command;
+        while ($granted->valid()) {
+            // Chunk objects to avoid timeouts with large sets
+            yield $command->setObjects(new LimitIterator(new NoRewindIterator($granted), 0, 250));
         }
     }
 }

application/forms/Command/Object/AddCommentForm.php

@@ -20,6 +20,8 @@ use ipl\Validator\CallbackValidator;
 use ipl\Web\FormDecorator\IcingaFormDecorator;
 use ipl\Web\Widget\Icon;
 use Iterator;
+use LimitIterator;
+use NoRewindIterator;
 use Traversable;
 
 use function ipl\Stdlib\iterable_value_first;
@@ -136,8 +138,9 @@ class AddCommentForm extends CommandForm
             'submit',
             'btn_submit',
             [
-                'required'  => true,
-                'label'     => tp('Add comment', 'Add comments', count($this->getObjects()))
+                'required'              => true,
+                'label'                 => tp('Add comment', 'Add comments', count($this->getObjects())),
+                'data-progress-label'   => tp('Adding comment', 'Adding comments', count($this->getObjects()))
             ]
         );
 
@@ -150,19 +153,19 @@ class AddCommentForm extends CommandForm
             return $this->isGrantedOn('icingadb/command/comment/add', $object);
         });
 
+        $command = new AddCommentCommand();
+        $command->setComment($this->getValue('comment'));
+        $command->setAuthor($this->getAuth()->getUser()->getUsername());
+
+        if (($expireTime = $this->getValue('expire_time'))) {
+            /** @var DateTime $expireTime */
+            $command->setExpireTime($expireTime->getTimestamp());
+        }
+
         $granted->rewind(); // Forwards the pointer to the first element
-        if ($granted->valid()) {
-            $command = new AddCommentCommand();
-            $command->setObjects($granted);
-            $command->setComment($this->getValue('comment'));
-            $command->setAuthor($this->getAuth()->getUser()->getUsername());
-
-            if (($expireTime = $this->getValue('expire_time'))) {
-                /** @var DateTime $expireTime */
-                $command->setExpireTime($expireTime->getTimestamp());
-            }
-
-            yield $command;
+        while ($granted->valid()) {
+            // Chunk objects to avoid timeouts with large sets
+            yield $command->setObjects(new LimitIterator(new NoRewindIterator($granted), 0, 500));
         }
     }
 }

application/forms/Command/Object/CheckNowForm.php

@@ -11,6 +11,8 @@ use Icinga\Web\Notification;
 use ipl\Orm\Model;
 use ipl\Web\Widget\Icon;
 use Iterator;
+use LimitIterator;
+use NoRewindIterator;
 use Traversable;
 
 class CheckNowForm extends CommandForm
@@ -56,14 +58,14 @@ class CheckNowForm extends CommandForm
                 );
         });
 
-        $granted->rewind(); // Forwards the pointer to the first element
-        if ($granted->valid()) {
-            $command = new ScheduleCheckCommand();
-            $command->setObjects($granted);
-            $command->setCheckTime(time());
-            $command->setForced();
+        $command = new ScheduleCheckCommand();
+        $command->setCheckTime(time());
+        $command->setForced();
 
-            yield $command;
+        $granted->rewind(); // Forwards the pointer to the first element
+        while ($granted->valid()) {
+            // Chunk objects to avoid timeouts with large sets
+            yield $command->setObjects(new LimitIterator(new NoRewindIterator($granted), 0, 1000));
         }
     }
 }

application/forms/Command/Object/DeleteCommentForm.php

@@ -11,6 +11,8 @@ use Icinga\Web\Notification;
 use ipl\Orm\Model;
 use ipl\Web\Widget\Icon;
 use Iterator;
+use LimitIterator;
+use NoRewindIterator;
 use Traversable;
 
 class DeleteCommentForm extends CommandForm
@@ -58,13 +60,13 @@ class DeleteCommentForm extends CommandForm
             return $this->isGrantedOn('icingadb/command/comment/delete', $object->{$object->object_type});
         });
 
-        $granted->rewind(); // Forwards the pointer to the first element
-        if ($granted->valid()) {
-            $command = new DeleteCommentCommand();
-            $command->setObjects($granted);
-            $command->setAuthor($this->getAuth()->getUser()->getUsername());
+        $command = new DeleteCommentCommand();
+        $command->setAuthor($this->getAuth()->getUser()->getUsername());
 
-            yield $command;
+        $granted->rewind(); // Forwards the pointer to the first element
+        while ($granted->valid()) {
+            // Chunk objects to avoid timeouts with large sets
+            yield $command->setObjects(new LimitIterator(new NoRewindIterator($granted), 0, 500));
         }
     }
 }

application/forms/Command/Object/DeleteDowntimeForm.php

@@ -11,6 +11,8 @@ use Icinga\Web\Notification;
 use ipl\Orm\Model;
 use ipl\Web\Widget\Icon;
 use Iterator;
+use LimitIterator;
+use NoRewindIterator;
 use Traversable;
 
 class DeleteDowntimeForm extends CommandForm
@@ -71,13 +73,13 @@ class DeleteDowntimeForm extends CommandForm
                 && $this->isGrantedOn('icingadb/command/downtime/delete', $object->{$object->object_type});
         });
 
-        $granted->rewind(); // Forwards the pointer to the first element
-        if ($granted->valid()) {
-            $command = new DeleteDowntimeCommand();
-            $command->setObjects($granted);
-            $command->setAuthor($this->getAuth()->getUser()->getUsername());
+        $command = new DeleteDowntimeCommand();
+        $command->setAuthor($this->getAuth()->getUser()->getUsername());
 
-            yield $command;
+        $granted->rewind(); // Forwards the pointer to the first element
+        while ($granted->valid()) {
+            // Chunk objects to avoid timeouts with large sets
+            yield $command->setObjects(new LimitIterator(new NoRewindIterator($granted), 0, 250));
         }
     }
 }

application/forms/Command/Object/ProcessCheckResultForm.php

@@ -16,6 +16,8 @@ use ipl\Orm\Model;
 use ipl\Web\FormDecorator\IcingaFormDecorator;
 use ipl\Web\Widget\Icon;
 use Iterator;
+use LimitIterator;
+use NoRewindIterator;
 use Traversable;
 
 use function ipl\Stdlib\iterable_value_first;
@@ -127,6 +129,11 @@ class ProcessCheckResultForm extends CommandForm
                     'Submit Passive Check Result',
                     'Submit Passive Check Results',
                     count($this->getObjects())
+                ),
+                'data-progress-label' => tp(
+                    'Submitting Passive Check Result',
+                    'Submitting Passive Check Results',
+                    count($this->getObjects())
                 )
             ]
         );
@@ -141,15 +148,15 @@ class ProcessCheckResultForm extends CommandForm
                 && $this->isGrantedOn('icingadb/command/process-check-result', $object);
         });
 
-        $granted->rewind(); // Forwards the pointer to the first element
-        if ($granted->valid()) {
-            $command = new ProcessCheckResultCommand();
-            $command->setObjects($granted);
-            $command->setStatus($this->getValue('status'));
-            $command->setOutput($this->getValue('output'));
-            $command->setPerformanceData($this->getValue('perfdata'));
+        $command = new ProcessCheckResultCommand();
+        $command->setStatus($this->getValue('status'));
+        $command->setOutput($this->getValue('output'));
+        $command->setPerformanceData($this->getValue('perfdata'));
 
-            yield $command;
+        $granted->rewind(); // Forwards the pointer to the first element
+        while ($granted->valid()) {
+            // Chunk objects to avoid timeouts with large sets
+            yield $command->setObjects(new LimitIterator(new NoRewindIterator($granted), 0, 250));
         }
     }
 }

application/forms/Command/Object/RemoveAcknowledgementForm.php

@@ -12,6 +12,8 @@ use Icinga\Web\Notification;
 use ipl\Orm\Model;
 use ipl\Web\Widget\Icon;
 use Iterator;
+use LimitIterator;
+use NoRewindIterator;
 use Traversable;
 
 use function ipl\Stdlib\iterable_value_first;
@@ -71,13 +73,13 @@ class RemoveAcknowledgementForm extends CommandForm
             return $this->isGrantedOn('icingadb/command/remove-acknowledgement', $object);
         });
 
-        $granted->rewind(); // Forwards the pointer to the first element
-        if ($granted->valid()) {
-            $command = new RemoveAcknowledgementCommand();
-            $command->setObjects($granted);
-            $command->setAuthor($this->getAuth()->getUser()->getUsername());
+        $command = new RemoveAcknowledgementCommand();
+        $command->setAuthor($this->getAuth()->getUser()->getUsername());
 
-            yield $command;
+        $granted->rewind(); // Forwards the pointer to the first element
+        while ($granted->valid()) {
+            // Chunk objects to avoid timeouts with large sets
+            yield $command->setObjects(new LimitIterator(new NoRewindIterator($granted), 0, 250));
         }
     }
 }

application/forms/Command/Object/ScheduleCheckForm.php

@@ -18,6 +18,8 @@ use ipl\Orm\Model;
 use ipl\Web\FormDecorator\IcingaFormDecorator;
 use ipl\Web\Widget\Icon;
 use Iterator;
+use LimitIterator;
+use NoRewindIterator;
 use Traversable;
 
 use function ipl\Stdlib\iterable_value_first;
@@ -103,8 +105,9 @@ class ScheduleCheckForm extends CommandForm
             'submit',
             'btn_submit',
             [
-                'required'  => true,
-                'label'     => tp('Schedule check', 'Schedule checks', count($this->getObjects()))
+                'required'              => true,
+                'label'                 => tp('Schedule check', 'Schedule checks', count($this->getObjects())),
+                'data-progress-label'   => tp('Scheduling check', 'Scheduling checks', count($this->getObjects()))
             ]
         );
 
@@ -121,14 +124,14 @@ class ScheduleCheckForm extends CommandForm
                 );
         });
 
-        $granted->rewind(); // Forwards the pointer to the first element
-        if ($granted->valid()) {
-            $command = new ScheduleCheckCommand();
-            $command->setObjects($granted);
-            $command->setForced($this->getElement('force_check')->isChecked());
-            $command->setCheckTime($this->getValue('check_time')->getTimestamp());
+        $command = new ScheduleCheckCommand();
+        $command->setForced($this->getElement('force_check')->isChecked());
+        $command->setCheckTime($this->getValue('check_time')->getTimestamp());
 
-            yield $command;
+        $granted->rewind(); // Forwards the pointer to the first element
+        while ($granted->valid()) {
+            // Chunk objects to avoid timeouts with large sets
+            yield $command->setObjects(new LimitIterator(new NoRewindIterator($granted), 0, 1000));
         }
     }
 }

application/forms/Command/Object/ScheduleServiceDowntimeForm.php

@@ -19,6 +19,8 @@ use ipl\Validator\CallbackValidator;
 use ipl\Web\FormDecorator\IcingaFormDecorator;
 use ipl\Web\Widget\Icon;
 use Iterator;
+use LimitIterator;
+use NoRewindIterator;
 use Traversable;
 
 class ScheduleServiceDowntimeForm extends CommandForm
@@ -258,8 +260,9 @@ class ScheduleServiceDowntimeForm extends CommandForm
             'submit',
             'btn_submit',
             [
-                'required'  => true,
-                'label'     => tp('Schedule downtime', 'Schedule downtimes', count($this->getObjects()))
+                'required'              => true,
+                'label'                 => tp('Schedule downtime', 'Schedule downtimes', count($this->getObjects())),
+                'data-progress-label'   => tp('Scheduling downtime', 'Scheduling downtimes', count($this->getObjects()))
             ]
         );
 
@@ -272,24 +275,24 @@ class ScheduleServiceDowntimeForm extends CommandForm
             return $this->isGrantedOn('icingadb/command/downtime/schedule', $object);
         });
 
+        $command = new ScheduleDowntimeCommand();
+        $command->setComment($this->getValue('comment'));
+        $command->setAuthor($this->getAuth()->getUser()->getUsername());
+        $command->setStart($this->getValue('start')->getTimestamp());
+        $command->setEnd($this->getValue('end')->getTimestamp());
+        $command->setChildOption((int) $this->getValue('child_options'));
+
+        if ($this->getElement('flexible')->isChecked()) {
+            $command->setFixed(false);
+            $command->setDuration(
+                $this->getValue('hours') * 3600 + $this->getValue('minutes') * 60
+            );
+        }
+
         $granted->rewind(); // Forwards the pointer to the first element
-        if ($granted->valid()) {
-            $command = new ScheduleDowntimeCommand();
-            $command->setObjects($granted);
-            $command->setComment($this->getValue('comment'));
-            $command->setAuthor($this->getAuth()->getUser()->getUsername());
-            $command->setStart($this->getValue('start')->getTimestamp());
-            $command->setEnd($this->getValue('end')->getTimestamp());
-            $command->setChildOption((int) $this->getValue('child_options'));
-
-            if ($this->getElement('flexible')->isChecked()) {
-                $command->setFixed(false);
-                $command->setDuration(
-                    $this->getValue('hours') * 3600 + $this->getValue('minutes') * 60
-                );
-            }
-
-            yield $command;
+        while ($granted->valid()) {
+            // Chunk objects to avoid timeouts with large sets
+            yield $command->setObjects(new LimitIterator(new NoRewindIterator($granted), 0, 250));
         }
     }
 }

application/forms/Command/Object/SendCustomNotificationForm.php

@@ -17,6 +17,8 @@ use ipl\Orm\Model;
 use ipl\Web\FormDecorator\IcingaFormDecorator;
 use ipl\Web\Widget\Icon;
 use Iterator;
+use LimitIterator;
+use NoRewindIterator;
 use Traversable;
 
 use function ipl\Stdlib\iterable_value_first;
@@ -104,7 +106,12 @@ class SendCustomNotificationForm extends CommandForm
             'btn_submit',
             [
                 'required'  => true,
-                'label'     => tp('Send custom notification', 'Send custom notifications', count($this->getObjects()))
+                'label'     => tp('Send custom notification', 'Send custom notifications', count($this->getObjects())),
+                'data-progress-label' => tp(
+                    'Sending custom notification',
+                    'Sending custom notifications',
+                    count($this->getObjects())
+                )
             ]
         );
 
@@ -117,15 +124,15 @@ class SendCustomNotificationForm extends CommandForm
             return $this->isGrantedOn('icingadb/command/send-custom-notification', $object);
         });
 
-        $granted->rewind(); // Forwards the pointer to the first element
-        if ($granted->valid()) {
-            $command = new SendCustomNotificationCommand();
-            $command->setObjects($granted);
-            $command->setComment($this->getValue('comment'));
-            $command->setForced($this->getElement('forced')->isChecked());
-            $command->setAuthor($this->getAuth()->getUser()->getUsername());
+        $command = new SendCustomNotificationCommand();
+        $command->setComment($this->getValue('comment'));
+        $command->setForced($this->getElement('forced')->isChecked());
+        $command->setAuthor($this->getAuth()->getUser()->getUsername());
 
-            yield $command;
+        $granted->rewind(); // Forwards the pointer to the first element
+        while ($granted->valid()) {
+            // Chunk objects to avoid timeouts with large sets
+            yield $command->setObjects(new LimitIterator(new NoRewindIterator($granted), 0, 500));
         }
     }
 }

application/forms/Command/Object/ToggleObjectFeaturesForm.php

@@ -12,6 +12,8 @@ use ipl\Html\FormElement\CheckboxElement;
 use ipl\Orm\Model;
 use ipl\Web\FormDecorator\IcingaFormDecorator;
 use Iterator;
+use LimitIterator;
+use NoRewindIterator;
 use Traversable;
 
 class ToggleObjectFeaturesForm extends CommandForm
@@ -25,7 +27,7 @@ class ToggleObjectFeaturesForm extends CommandForm
     /**
      * ToggleFeature(s) being used to submit this form
      *
-     * @var ToggleObjectFeatureCommand[]
+     * @var array<string, bool>
      */
     protected $submittedFeatures = [];
 
@@ -62,9 +64,8 @@ class ToggleObjectFeaturesForm extends CommandForm
                 return;
             }
 
-            foreach ($this->submittedFeatures as $feature) {
-                $enabled = $feature->getEnabled();
-                switch ($feature->getFeature()) {
+            foreach ($this->submittedFeatures as $feature => $enabled) {
+                switch ($feature) {
                     case ToggleObjectFeatureCommand::FEATURE_ACTIVE_CHECKS:
                         if ($enabled) {
                             $message = t('Enabled active checks successfully');
@@ -175,16 +176,16 @@ class ToggleObjectFeaturesForm extends CommandForm
                 return $this->isGrantedOn($spec['permission'], $object);
             });
 
+            $command = new ToggleObjectFeatureCommand();
+            $command->setFeature($feature);
+            $command->setEnabled((int) $state);
+
             $granted->rewind(); // Forwards the pointer to the first element
-            if ($granted->valid()) {
-                $command = new ToggleObjectFeatureCommand();
-                $command->setObjects($granted);
-                $command->setFeature($feature);
-                $command->setEnabled((int) $state);
+            while ($granted->valid()) {
+                $this->submittedFeatures[$command->getFeature()] ??= $command->getEnabled();
 
-                $this->submittedFeatures[] = $command;
-
-                yield $command;
+                // Chunk objects to avoid timeouts with large sets
+                yield $command->setObjects(new LimitIterator(new NoRewindIterator($granted), 0, 1000));
             }
         }
     }

library/Icingadb/Command/Transport/ApiCommandTransport.php

@@ -19,6 +19,9 @@ use Icinga\Util\Json;
  */
 class ApiCommandTransport implements CommandTransportInterface
 {
+    /** @var int Used timeout when sending a command */
+    protected const SEND_TIMEOUT = 15;
+
     /**
      * Transport identifier
      */
@@ -237,7 +240,7 @@ class ApiCommandTransport implements CommandTransportInterface
         }
 
         try {
-            $response = (new Client(['timeout' => 15]))
+            $response = (new Client(['timeout' => static::SEND_TIMEOUT]))
                 ->post($this->getUriFor($command->getEndpoint()), [
                     'auth'          => [$this->getUsername(), $this->getPassword()],
                     'headers'       => $headers,
@@ -246,6 +249,13 @@ class ApiCommandTransport implements CommandTransportInterface
                     'verify'        => false
                 ]);
         } catch (GuzzleException $e) {
+            if (str_starts_with(ltrim($e->getMessage()), 'cURL error 28:')) {
+                throw new ApiCommandException(t(
+                    'No response from the Icinga 2 API received after %d seconds.'
+                    . ' Please make sure the action has not been performed, before retrying'
+                ), static::SEND_TIMEOUT, $e);
+            }
+
             throw new CommandTransportException(
                 'Can\'t connect to the Icinga 2 API: %u %s',
                 $e->getCode(),
@@ -311,7 +321,7 @@ class ApiCommandTransport implements CommandTransportInterface
     public function probe()
     {
         try {
-            $response = (new Client(['timeout' => 15]))
+            $response = (new Client(['timeout' => static::SEND_TIMEOUT]))
                 ->get($this->getUriFor(''), [
                     'auth'          => [$this->getUsername(), $this->getPassword()],
                     'headers'       => ['Accept' => 'application/json'],

library/Icingadb/Common/CommandActions.php

@@ -17,6 +17,7 @@ use Icinga\Module\Icingadb\Forms\Command\Object\ScheduleServiceDowntimeForm;
 use Icinga\Module\Icingadb\Forms\Command\Object\SendCustomNotificationForm;
 use Icinga\Module\Icingadb\Forms\Command\Object\ToggleObjectFeaturesForm;
 use Icinga\Security\SecurityException;
+use Icinga\Util\Environment;
 use Icinga\Web\Notification;
 use ipl\Orm\Model;
 use ipl\Orm\Query;
@@ -151,6 +152,10 @@ trait CommandActions
             $this->httpBadRequest('Responding with JSON during a Web request is not supported');
         }
 
+        // Bulk operations may require more memory and time
+        Environment::raiseMemoryLimit();
+        Environment::raiseExecutionTime();
+
         if (is_string($form)) {
             /** @var CommandForm $form */
             $form = new $form();