plugins/BotTracking/tests/Integration/TrackerTest.php

<?php

/**
 * Matomo - free/libre analytics platform
 *
 * @link    https://matomo.org
 * @license https://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
 */

declare(strict_types=1);

namespace Piwik\Plugins\BotTracking\tests\Integration;

use Piwik\Common;
use Piwik\Db;
use Piwik\Plugins\BotTracking\BotDetector;
use Piwik\Plugins\BotTracking\Dao\BotRequestsDao;
use Piwik\Tests\Framework\Fixture;
use Piwik\Tests\Framework\TestCase\IntegrationTestCase;

/**
 * @group BotTracking
 * @group Plugins
 */
class TrackerTest extends IntegrationTestCase
{
    public function setUp(): void
    {
        parent::setUp();

        Fixture::createSuperUser();
        Fixture::createWebsite('2014-02-04');
    }

    /**
     * @dataProvider getBotUserAgents
     */
    public function testAIChatbotIsTrackedCorrectly(string $userAgent, string $expectedAgent, string $expectedType): void
    {
        $t = Fixture::getTracker(1, '2025-02-02 12:00:00');
        $t->setUserAgent($userAgent);
        $t->setUrl('https://matomo.org/faq/123');
        $t->setCustomTrackingParameter('recMode', '1');

        Fixture::checkResponse($t->doTrackPageView(''));

        $tableName = BotRequestsDao::getPrefixedTableName();
        $sql = "SELECT * FROM `{$tableName}` WHERE idsite = ?";
        $records = Db::fetchAll($sql, [1]);

        self::assertCount(1, $records);
        self::assertEquals($expectedAgent, $records[0]['bot_name']);
        self::assertEquals($expectedType, $records[0]['bot_type']);

        $tableName = Common::prefixTable('log_visit');
        $sql = "SELECT COUNT(*) FROM `{$tableName}` WHERE idsite = ?";
        self::assertEquals(0, Db::fetchOne($sql, [1]));

        $tableName = Common::prefixTable('log_action');
        $sql = "SELECT COUNT(*) FROM `{$tableName}`";
        self::assertEquals(1, Db::fetchOne($sql));
    }

    /**
     * @return array<array{0: string, 1: string, 2: string}>
     */
    public function getBotUserAgents(): array
    {
        return [
            ['ChatGPT-User/1.0', 'ChatGPT-User', BotDetector::BOT_TYPE_AI_CHATBOT],
            ['chatgpt-user/1.0', 'ChatGPT-User', BotDetector::BOT_TYPE_AI_CHATBOT],
            ['CHATGPT-USER/1.0', 'ChatGPT-User', BotDetector::BOT_TYPE_AI_CHATBOT],
            ['Mozilla/5.0 (compatible; ChatGPT-User/1.0)', 'ChatGPT-User', BotDetector::BOT_TYPE_AI_CHATBOT],
            ['Mozilla/5.0 (compatible; ChatGPT-User/1.0; +https://openai.com)', 'ChatGPT-User', BotDetector::BOT_TYPE_AI_CHATBOT],
            ['MistralAI-User/2.0', 'MistralAI-User', BotDetector::BOT_TYPE_AI_CHATBOT],
            ['Gemini-Deep-Research/1.0', 'Gemini-Deep-Research', BotDetector::BOT_TYPE_AI_CHATBOT],
            ['Claude-User/3.0', 'Claude-User', BotDetector::BOT_TYPE_AI_CHATBOT],
            ['Perplexity-User/1.0', 'Perplexity-User', BotDetector::BOT_TYPE_AI_CHATBOT],
            ['Google-NotebookLM/1.0', 'Google-NotebookLM', BotDetector::BOT_TYPE_AI_CHATBOT],
        ];
    }

    public function testActionIdsAreCorrectlyReused(): void
    {
        $t = Fixture::getTracker(1, '2025-02-02 12:00:00');
        $t->setUserAgent('ChatGPT-User/1.0');
        $t->setUrl('https://matomo.org/faq/123');
        $t->setCustomTrackingParameter('recMode', '1');

        Fixture::checkResponse($t->doTrackPageView(''));

        $tableName = BotRequestsDao::getPrefixedTableName();
        $sql = "SELECT * FROM `{$tableName}` WHERE idsite = ?";
        $records = Db::fetchAll($sql, [1]);

        self::assertCount(1, $records);
        self::assertEquals('ChatGPT-User', $records[0]['bot_name']);
        self::assertEquals(BotDetector::BOT_TYPE_AI_CHATBOT, $records[0]['bot_type']);

        $tableName = Common::prefixTable('log_visit');
        $sql = "SELECT COUNT(*) FROM `{$tableName}` WHERE idsite = ?";
        self::assertEquals(0, Db::fetchOne($sql, [1]));

        $tableName = Common::prefixTable('log_action');
        $sql = "SELECT COUNT(*) FROM `{$tableName}`";
        self::assertEquals(1, Db::fetchOne($sql));

        // track another bot request to ensure now new action is created
        $t = Fixture::getTracker(1, '2025-02-02 12:00:00');
        $t->setUserAgent('Gemini-Deep-Research/1.0');
        $t->setUrl('https://matomo.org/faq/123');
        $t->setCustomTrackingParameter('recMode', '1');

        Fixture::checkResponse($t->doTrackPageView(''));

        $tableName = BotRequestsDao::getPrefixedTableName();
        $sql = "SELECT * FROM `{$tableName}` WHERE idsite = ?";
        $records = Db::fetchAll($sql, [1]);

        self::assertCount(2, $records);
        self::assertEquals('Gemini-Deep-Research', $records[1]['bot_name']);
        self::assertEquals(BotDetector::BOT_TYPE_AI_CHATBOT, $records[1]['bot_type']);

        $tableName = Common::prefixTable('log_visit');
        $sql = "SELECT COUNT(*) FROM `{$tableName}` WHERE idsite = ?";
        self::assertEquals(0, Db::fetchOne($sql, [1]));

        $tableName = Common::prefixTable('log_action');
        $sql = "SELECT COUNT(*) FROM `{$tableName}`";
        self::assertEquals(1, Db::fetchOne($sql));
    }

    public function testVisitsAndBotsShareActions(): void
    {
        // track a normal visit
        $t = Fixture::getTracker(1, '2025-02-02 12:00:00');
        $t->setUrl('https://matomo.org/faq/123');
        Fixture::checkResponse($t->doTrackPageView(''));

        $tableName = BotRequestsDao::getPrefixedTableName();
        $sql       = "SELECT * FROM `{$tableName}` WHERE idsite = ?";
        $records   = Db::fetchAll($sql, [1]);
        self::assertCount(0, $records);

        $tableName = Common::prefixTable('log_visit');
        $sql       = "SELECT COUNT(*) FROM `{$tableName}` WHERE idsite = ?";
        self::assertEquals(1, Db::fetchOne($sql, [1]));

        $tableName = Common::prefixTable('log_action');
        $sql       = "SELECT COUNT(*) FROM `{$tableName}`";
        self::assertEquals(1, Db::fetchOne($sql));

        // track another bot request to ensure now new action is created
        $t = Fixture::getTracker(1, '2025-02-02 12:00:00');
        $t->setUserAgent('Gemini-Deep-Research/1.0');
        $t->setUrl('https://matomo.org/faq/123');
        $t->setCustomTrackingParameter('recMode', '1');

        Fixture::checkResponse($t->doTrackPageView(''));

        $tableName = BotRequestsDao::getPrefixedTableName();
        $sql       = "SELECT * FROM `{$tableName}` WHERE idsite = ?";
        $records   = Db::fetchAll($sql, [1]);

        self::assertCount(1, $records);
        self::assertEquals('Gemini-Deep-Research', $records[0]['bot_name']);
        self::assertEquals(BotDetector::BOT_TYPE_AI_CHATBOT, $records[0]['bot_type']);

        $tableName = Common::prefixTable('log_visit');
        $sql       = "SELECT COUNT(*) FROM `{$tableName}` WHERE idsite = ?";
        self::assertEquals(1, Db::fetchOne($sql, [1]));

        $tableName = Common::prefixTable('log_action');
        $sql       = "SELECT COUNT(*) FROM `{$tableName}`";
        self::assertEquals(1, Db::fetchOne($sql));
    }

    public function testVisitsAndBotsShareActionsWhenQueryParametersAreExcluded(): void
    {
        $idSite = Fixture::createWebsite(
            '2014-02-04',
            $ecommerce = 0,
            $siteName = false,
            $siteUrl = false,
            $siteSearch = 1,
            $searchKeywordParameters = null,
            $searchCategoryParameters = null,
            $timezone = null,
            $type = null,
            $excludeUnknownUrls = 0,
            $excludedParameters = 'excluded'
        );

        $url = 'https://matomo.org/faq/123?keep=1&excluded=secret';

        // track a normal visit with the excluded parameter
        $t = Fixture::getTracker($idSite, '2025-02-02 12:00:00');
        $t->setUrl($url);
        Fixture::checkResponse($t->doTrackPageView(''));

        // track a bot request with the same URL
        $t = Fixture::getTracker($idSite, '2025-02-02 12:00:00');
        $t->setUserAgent('Gemini-Deep-Research/1.0');
        $t->setUrl($url);
        $t->setCustomTrackingParameter('recMode', '1');
        Fixture::checkResponse($t->doTrackPageView(''));

        // the bot request must store the same cleaned URL the normal visit stored,
        // so both resolve to a single shared action
        $tableName = BotRequestsDao::getPrefixedTableName();
        $idActionUrl = Db::fetchOne("SELECT idaction_url FROM `{$tableName}` WHERE idsite = ?", [$idSite]);

        $actionTable = Common::prefixTable('log_action');
        $name = Db::fetchOne("SELECT name FROM `{$actionTable}` WHERE idaction = ?", [$idActionUrl]);
        self::assertEquals('matomo.org/faq/123?keep=1', $name);

        $count = Db::fetchOne("SELECT COUNT(*) FROM `{$actionTable}` WHERE name = ? AND type = ?", [
            'matomo.org/faq/123?keep=1',
            \Piwik\Tracker\Action::TYPE_PAGE_URL,
        ]);
        self::assertEquals(1, $count);
    }
}