Adds detail report around requests made by AI bots (#23793)

* Adds archiving for new report(s)

* Add API methods for new report

* Add report classes

* Move AI Agent reports to bottom

* Add system tests for new report

* Adds UI tests for new report

* updates expected UI test files

* fix other tests

* do not double count acquired visits

* remove explicit plugin loading from ui tests

* updates expected UI test file

Author: sgiehl

config/global.ini.php

@@ -632,6 +632,11 @@
 ; maximum number of rows for the Products reports
 datatable_archiving_maximum_rows_products = 10000
 
+; maximum number of AI Assistants listed in Bot Tracking reports
+datatable_archiving_maximum_rows_bots = 250
+; maximum number of page/document rows listed per AI Assistant in Bot Tracking reports
+datatable_archiving_maximum_rows_subtable_bots = 250
+
 ; maximum number of rows for other tables (Providers, User settings configurations)
 datatable_archiving_maximum_rows_standard = 500
 

plugins/AIAgents/Reports/Get.php

@@ -33,7 +33,7 @@ protected function init()
         $this->name          = Piwik::translate('AIAgents_AIAgentVisits');
         $this->categoryId    = 'AIAgents_AIAssistants';
         $this->subcategoryId = 'General_Overview';
-        $this->order         = 10;
+        $this->order         = 90;
 
         $this->processedMetrics = [
             new AIAgentMetric(new AverageTimeOnSite(), API::AI_AGENT_COLUMN_SUFFIX),
@@ -66,14 +66,14 @@ public function configureWidgets(WidgetsList $widgetsList, ReportWidgetFactory $
                 ->setName('AIAgents_WidgetGraphAIAgents')
                 ->forceViewDataTable(Evolution::ID)
                 ->setAction('getEvolutionGraph')
-                ->setOrder(1)
+                ->setOrder(90)
         );
 
         $widgetsList->addWidgetConfig(
             $factory->createWidget()
                 ->forceViewDataTable(Sparklines::ID)
                 ->setName('AIAgents_WidgetOverviewAIAgents')
-                ->setOrder(2)
+                ->setOrder(91)
         );
     }
 

plugins/AIAgents/tests/UI/AIAgents_spec.js

@@ -17,7 +17,7 @@ describe('AIAgents', function () {
     await page.waitForNetworkIdle();
 
     const widgets = await page.$$('.matomo-widget');
-    expect(widgets.length).to.equal(2);
+    expect(widgets.length).to.equal(3);
   });
 
   it('should show the AI assistants report menu items', async function () {

plugins/AIAgents/tests/UI/expected-ui-screenshots/AIAgents_overview.png

@@ -0,0 +1,76 @@
+<?php
+
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link    https://matomo.org
+ * @license https://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+declare(strict_types=1);
+
+namespace Piwik\Plugins\BotTracking;
+
+use Piwik\Archive;
+use Piwik\DataTable;
+use Piwik\DataTable\DataTableInterface;
+use Piwik\Piwik;
+
+class API extends \Piwik\Plugin\API
+{
+    /**
+     * Returns a report about AI assistants crawling your site and how many hits each one generates. Depending on the provided secondary dimension
+     * the subtable will either contain all requested page urls or document urls.
+     *
+     * @param string|int|int[] $idSite
+     * @param null|'pages'|'documents' $secondaryDimension can be either `pages` (default) or `documents`
+     * @return DataTable|DataTable\Map
+     */
+    public function getAIAssistantRequests($idSite, string $period, string $date, bool $expanded = false, bool $flat = false, ?string $secondaryDimension = null): DataTableInterface
+    {
+        Piwik::checkUserHasViewAccess($idSite);
+
+        $archiveName = Archiver::AI_ASSISTANTS_PAGES_RECORD;
+
+        if ($secondaryDimension === 'documents') {
+            $archiveName = Archiver::AI_ASSISTANTS_DOCUMENTS_RECORD;
+        }
+
+        $dataTable = Archive::createDataTableFromArchive($archiveName, $idSite, $period, $date, '', $expanded, $flat);
+
+        // When flattening a report, remove all main table rows, where no subtable exists
+        if ($flat) {
+            $dataTable->filter(function (DataTable $table) {
+                foreach ($table->getRows() as $key => $row) {
+                    if (!$row->getIdSubDataTable()) {
+                        $table->deleteRow($key);
+                    }
+                }
+            });
+        }
+
+        return $dataTable;
+    }
+
+    /**
+     * @param string|int|int[] $idSite
+     * @return DataTable|DataTable\Map
+     */
+    public function getPageUrlsForAIAssistant($idSite, string $period, string $date, int $idSubtable): DataTableInterface
+    {
+        Piwik::checkUserHasViewAccess($idSite);
+
+        return Archive::createDataTableFromArchive(Archiver::AI_ASSISTANTS_PAGES_RECORD, $idSite, $period, $date, '', false, false, $idSubtable);
+    }
+
+    /**
+     * @param string|int|int[] $idSite
+     * @return DataTable|DataTable\Map
+     */
+    public function getDocumentUrlsForAIAssistant($idSite, string $period, string $date, int $idSubtable): DataTableInterface
+    {
+        Piwik::checkUserHasViewAccess($idSite);
+
+        return Archive::createDataTableFromArchive(Archiver::AI_ASSISTANTS_DOCUMENTS_RECORD, $idSite, $period, $date, '', false, false, $idSubtable);
+    }
+}

plugins/BotTracking/API.php

@@ -0,0 +1,23 @@
+<?php
+
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link    https://matomo.org
+ * @license https://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+declare(strict_types=1);
+
+namespace Piwik\Plugins\BotTracking;
+
+class Archiver extends \Piwik\Plugin\Archiver
+{
+    public const AI_ASSISTANTS_PAGES_RECORD = 'BotTracking_AIAssistantsPages';
+    public const AI_ASSISTANTS_DOCUMENTS_RECORD = 'BotTracking_AIAssistantsDocuments';
+
+    public static function shouldRunEvenWhenNoVisits(): bool
+    {
+        return true;
+    }
+}

plugins/BotTracking/Archiver.php

@@ -0,0 +1,20 @@
+<?php
+
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link    https://matomo.org
+ * @license https://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+declare(strict_types=1);
+
+namespace Piwik\Plugins\BotTracking\Columns;
+
+use Piwik\Columns\Dimension;
+
+class AIAssistantName extends Dimension
+{
+    protected $nameSingular = 'BotTracking_ColumnAIAssistantName';
+    protected $type = self::TYPE_TEXT;
+}

plugins/BotTracking/Columns/AIAssistantName.php

@@ -0,0 +1,20 @@
+<?php
+
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link    https://matomo.org
+ * @license https://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+declare(strict_types=1);
+
+namespace Piwik\Plugins\BotTracking\Columns;
+
+use Piwik\Columns\Dimension;
+
+class DocumentUrl extends Dimension
+{
+    protected $nameSingular = 'BotTracking_DocumentUrl';
+    protected $type = self::TYPE_URL;
+}

plugins/BotTracking/Columns/DocumentUrl.php

@@ -0,0 +1,46 @@
+<?php
+
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link    https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+declare(strict_types=1);
+
+namespace Piwik\Plugins\BotTracking\Columns\Metrics;
+
+use Piwik\Columns\Dimension;
+use Piwik\Metrics\Formatter;
+use Piwik\Piwik;
+use Piwik\Plugin\AggregatedMetric;
+use Piwik\Plugins\BotTracking\Metrics;
+
+class AcquiredVisits extends AggregatedMetric
+{
+    public function getName()
+    {
+        return Metrics::COLUMN_ACQUIRED_VISITS;
+    }
+
+    public function getTranslatedName()
+    {
+        return Piwik::translate('BotTracking_ColumnAcquiredVisits');
+    }
+
+    public function getDocumentation()
+    {
+        return Piwik::translate('BotTracking_ColumnAcquiredVisitsDocumentation');
+    }
+
+    public function format($value, Formatter $formatter)
+    {
+        return $formatter->getPrettyNumber($value);
+    }
+
+    public function getSemanticType(): ?string
+    {
+        return Dimension::TYPE_NUMBER;
+    }
+}

plugins/BotTracking/Columns/Metrics/AcquiredVisits.php

@@ -0,0 +1,46 @@
+<?php
+
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link    https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+declare(strict_types=1);
+
+namespace Piwik\Plugins\BotTracking\Columns\Metrics;
+
+use Piwik\Columns\Dimension;
+use Piwik\Metrics\Formatter;
+use Piwik\Piwik;
+use Piwik\Plugin\AggregatedMetric;
+use Piwik\Plugins\BotTracking\Metrics;
+
+class DocumentRequests extends AggregatedMetric
+{
+    public function getName()
+    {
+        return Metrics::COLUMN_DOCUMENT_REQUESTS;
+    }
+
+    public function getTranslatedName()
+    {
+        return Piwik::translate('BotTracking_ColumnDocumentRequests');
+    }
+
+    public function getDocumentation()
+    {
+        return Piwik::translate('BotTracking_ColumnDocumentRequestsDocumentation');
+    }
+
+    public function format($value, Formatter $formatter)
+    {
+        return $formatter->getPrettyNumber($value);
+    }
+
+    public function getSemanticType(): ?string
+    {
+        return Dimension::TYPE_NUMBER;
+    }
+}