idSite, '2012-08-09 16:00:00'); if ($ip) { $t->setIp($ip); } $t->setUserAgent('Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148'); $t->setUrl('http://pastwebsite.com/here/we/go'); Fixture::checkResponse($t->doTrackPageView('a page title')); // update ts_archived for archives in 2012_08 so they will be invalidated and rearchived $sql = "UPDATE " . ArchiveTableCreator::getNumericTable(Date::factory('2012-08-01'), true) . " SET ts_archived = ?"; Db::query($sql, [Date::now()->subHour(2)->getDatetime()]); } private static function addNewSegmentToPast() { Config::getInstance()->General['enable_browser_archiving_triggering'] = 0; self::$fixture->getTestEnvironment()->overrideConfig('General', 'enable_browser_archiving_triggering', 0); self::$fixture->getTestEnvironment()->save(); Rules::setBrowserTriggerArchiving(false); // add one segment and set it's created/updated time to some time in the past so we don't re-archive for it $idSegment = API::getInstance()->add(self::NEW_SEGMENT_NAME, self::NEW_SEGMENT, self::$fixture->idSite, $autoArchive = 1, $enabledAllUsers = 1); // add another segment w/ special encoded value (but do it through the API which will use a definition query param which is not // encoded the same as Request::getRawSegmentFromRequest() $url = Fixture::getTestRootUrl() . '?' . http_build_query([ 'module' => 'API', 'method' => 'SegmentEditor.add', 'name' => self::ENCODED_SEGMENT_NAME, 'definition' => self::ENCODED_SEGMENT, 'idSite' => self::$fixture->idSite, 'autoArchive' => 1, 'enabledAllUsers' => 1, 'format' => 'json', 'token_auth' => Fixture::getTokenAuth(), ]); self::assertStringContainsString(urlencode(self::ENCODED_SEGMENT), $url); $idSegment2 = Http::sendHttpRequest($url, 10); $idSegment2 = json_decode($idSegment2, $isAssoc = true); $idSegment2 = $idSegment2['value']; Config::getInstance()->General['enable_browser_archiving_triggering'] = 1; self::$fixture->getTestEnvironment()->overrideConfig('General', 'enable_browser_archiving_triggering', 1); self::$fixture->getTestEnvironment()->save(); Rules::setBrowserTriggerArchiving(true); Db::exec("UPDATE " . Common::prefixTable('segment') . ' SET ts_created = \'2015-01-02 00:00:00\', ts_last_edit = \'2015-01-02 00:00:00\' WHERE idsegment IN (' . $idSegment . ", " . $idSegment2 . ")"); } private static function trackVisitsForToday() { $startTime = Date::today()->addHour(12)->getDatetime(); $t = Fixture::getTracker(self::$fixture->idSite, $startTime); $t->setUserAgent('Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148'); $t->setUrl('http://awebsite.com/here/we/go'); Fixture::checkResponse($t->doTrackPageView('a page title')); $t->setUrl('http://awebsite.com/another/page'); $t->setForceVisitDateTime(Date::factory($startTime)->addHour(1)); Fixture::checkResponse($t->doTrackPageView('a second page title')); } public function getApiForTesting() { $apiRequiringSegments = ['Goals.get', 'VisitFrequency.get']; $results = array(); foreach (self::$fixture->getDefaultSegments() as $segmentName => $info) { $results[] = array('VisitsSummary.get', array('idSite' => 'all', 'date' => '2012-08-09', 'periods' => array('day', 'week', 'month', 'year'), 'segment' => $info['definition'], 'testSuffix' => '_' . $segmentName)); } $results[] = array('VisitsSummary.get', array('idSite' => 'all', 'date' => 'today', 'periods' => ['day', 'week', 'month', 'year'], 'segment' => self::NEW_SEGMENT, 'testSuffix' => '_' . self::NEW_SEGMENT_NAME)); $results[] = array('VisitsSummary.get', array('idSite' => 'all', 'date' => 'today', 'periods' => ['day', 'week', 'month', 'year'], 'segment' => self::ENCODED_SEGMENT, 'testSuffix' => '_' . self::ENCODED_SEGMENT_NAME)); // ExamplePlugin metric $results[] = ['ExamplePlugin.getExampleArchivedMetric', [ 'idSite' => 'all', 'date' => '2007-04-05', 'periods' => ['day', 'week'], ]]; $results[] = ['Actions.get', [ 'idSite' => 'all', 'date' => '2007-04-05', 'periods' => ['day', 'week'], 'testSuffix' => '_examplePluginNoMetricsBecauseNoOtherPluginsArchived', ]]; // API Call Without segments $results[] = array('VisitsSummary.get', array('idSite' => 'all', 'date' => '2012-08-09', 'periods' => array('day', 'month', 'year', 'week'))); $results[] = array($apiRequiringSegments, array('idSite' => 'all', 'date' => '2012-08-09', 'periods' => array('month'))); $results[] = array('VisitsSummary.get', array('idSite' => 'all', 'date' => '2012-08-09', 'periods' => array('day', 'week', 'month', 'year'), 'segment' => 'browserCode==EP', 'testSuffix' => '_nonPreArchivedSegment')); $segments = array(ManySitesImportedLogs::SEGMENT_PRE_ARCHIVED, ManySitesImportedLogs::SEGMENT_PRE_ARCHIVED_CONTAINS_ENCODED, ); foreach ($segments as $index => $segment) { // Test with a pre-processed segment $results[] = array(array('VisitsSummary.get', 'Live.getLastVisitsDetails', 'VisitFrequency.get'), array('idSite' => '1', 'date' => '2012-08-09', 'periods' => array('day', 'year'), 'segment' => $segment, 'testSuffix' => '_preArchivedSegment' . $index, 'otherRequestParameters' => array( 'hideColumns' => 'latitude,longitude', ), 'xmlFieldsToRemove' => array( 'fingerprint', ), ), ); } return $results; } public function testArchivePhpCron() { self::$fixture->getTestEnvironment()->overrideConfig('General', 'enable_browser_archiving_triggering', 0); self::$fixture->getTestEnvironment()->overrideConfig('General', 'browser_archiving_disabled_enforce', 1); self::$fixture->getTestEnvironment()->save(); Config::getInstance()->General['enable_browser_archiving_triggering'] = 0; Config::getInstance()->General['browser_archiving_disabled_enforce'] = 1; Rules::setBrowserTriggerArchiving(false); $dateToTest = '2007-04-05'; // invalidate exampleplugin only archives in past $invalidator = StaticContainer::get(ArchiveInvalidator::class); $invalidator->markArchivesAsInvalidated( [1], [$dateToTest], 'day', new Segment('', [1]), false, false, 'ExamplePlugin' ); // track a visit in 2007-04-05 so it will fully archive (ExamplePlugin wouldn't archive without visit) $tracker = Fixture::getTracker(1, $dateToTest); $tracker->setUrl('http://example.com/test/url'); Fixture::checkResponse($tracker->doTrackPageView('abcdefg')); $invalidationEntries = $this->getInvalidatedArchiveTableEntries(); // there should be 4 invalidations (day, week, month, year) for ExamplePlugin self::assertCount(4, $invalidationEntries); self::assertEquals(['done.ExamplePlugin'], array_unique(array_column($invalidationEntries, 'name'))); // empty the list so nothing is invalidated during core:archive (so we only archive ExamplePlugin and not all plugins) $invalidator->forgetRememberedArchivedReportsToInvalidate(1, Date::factory($dateToTest)); $this->runArchivePhpCron(); $table = ArchiveTableCreator::getNumericTable(Date::factory($dateToTest), true); $doneFlags = Db::fetchAll("SELECT `period`, `date1`, `date2`, `value` FROM `$table` WHERE name LIKE 'done.ExamplePlugin%' ORDER BY `period` ASC , `date1` ASC, `date2` ASC"); $expectedDoneFlags = [ ['period' => Day::PERIOD_ID, 'date1' => '2007-04-05', 'date2' => '2007-04-05', 'value' => ArchiveWriter::DONE_OK], ['period' => Week::PERIOD_ID, 'date1' => '2007-04-02', 'date2' => '2007-04-08', 'value' => ArchiveWriter::DONE_OK], ['period' => Month::PERIOD_ID, 'date1' => '2007-04-01', 'date2' => '2007-04-30', 'value' => ArchiveWriter::DONE_OK], // Year flag is stored in other table ]; self::assertEquals($expectedDoneFlags, $doneFlags); // add new segment w/ edited created/edit time so it will not trigger segment re-archiving, then track a visit // so the segments will be archived w/ other invalidation. this also runs core:archive forcing CURL requests. try { self::forceCurlCliMulti(); self::addNewSegmentToPast(); self::trackVisitsForToday(); self::trackVisitInPast(); // tracks visit on 2012-08-09, so we end up invalidating and rearchiving a past month $output = $this->runArchivePhpCron(); } finally { self::undoForceCurlCliMulti(); } // re archiving should not produce any ExamplePlugin specific archives, but 382 done flags for certain periods and segments $table = ArchiveTableCreator::getNumericTable(Date::factory('2012-08-09'), true); self::assertEquals(0, Db::fetchOne("SELECT count(*) FROM `$table` WHERE name LIKE 'done.ExamplePlugin%'")); self::assertEquals(382, Db::fetchOne("SELECT count(*) FROM `$table` WHERE name LIKE 'done%'")); foreach ($this->getApiForTesting() as $testInfo) { [$api, $params] = $testInfo; if (!isset($params['testSuffix'])) { $params['testSuffix'] = ''; } $params['testSuffix'] .= '_noOptions'; $params['disableArchiving'] = true; $success = $this->runApiTests($api, $params); if (!$success) { var_dump($output); } } } /** * @depends testArchivePhpCron */ public function testArchivePhpCronWithSingleReportRearchive() { self::$fixture->getTestEnvironment()->overrideConfig('General', 'enable_browser_archiving_triggering', 0); self::$fixture->getTestEnvironment()->overrideConfig('General', 'browser_archiving_disabled_enforce', 1); self::$fixture->getTestEnvironment()->save(); Config::getInstance()->General['enable_browser_archiving_triggering'] = 0; Config::getInstance()->General['browser_archiving_disabled_enforce'] = 1; Rules::setBrowserTriggerArchiving(false); // request a different date than other tests so we can test that only the single requested metric is archived $dateToRequest = '2007-04-04'; // track a visit so archiving will go through $tracker = Fixture::getTracker(1, $dateToRequest); $tracker->setUrl('http://example.com/test/url2'); Fixture::checkResponse($tracker->doTrackPageView('jkl')); $invalidator = StaticContainer::get(ArchiveInvalidator::class); $invalidator->forgetRememberedArchivedReportsToInvalidate(1, Date::factory($dateToRequest)); $table = ArchiveTableCreator::getNumericTable(Date::factory($dateToRequest), true); // three whole plugin archives from previous tests $doneFlags = Db::fetchAll("SELECT `period`, `date1`, `date2`, `value` FROM `$table` WHERE name LIKE 'done.ExamplePlugin%' ORDER BY `period` ASC , `date1` ASC, `date2` ASC, `value` ASC"); $expectedDoneFlags = [ ['period' => Day::PERIOD_ID, 'date1' => '2007-04-05', 'date2' => '2007-04-05', 'value' => ArchiveWriter::DONE_OK], ['period' => Week::PERIOD_ID, 'date1' => '2007-04-02', 'date2' => '2007-04-08', 'value' => ArchiveWriter::DONE_OK], ['period' => Month::PERIOD_ID, 'date1' => '2007-04-01', 'date2' => '2007-04-30', 'value' => ArchiveWriter::DONE_OK], // Year flag is stored in other table ]; self::assertEquals($expectedDoneFlags, $doneFlags); // invalidate a report so we get a partial archive (using the metric that gets incremented each time it is archived) // (do it after the last run so we don't end up just re-using the ExamplePlugin archive) $invalidator->markArchivesAsInvalidated([1], [$dateToRequest], 'day', new Segment('', [1]), false, false, 'ExamplePlugin.ExamplePlugin_example_metric'); $output = $this->runArchivePhpCron(['-vvv' => null]); $this->runApiTests('ExamplePlugin.getExampleArchivedMetric', [ 'idSite' => 'all', 'date' => $dateToRequest, 'periods' => ['day', 'week'], 'testSuffix' => '_singleMetric', ]); // test that latest archives for ExamplePlugin are partial $doneFlags = Db::fetchAll("SELECT `period`, `date1`, `date2`, `value` FROM `$table` WHERE name LIKE 'done.ExamplePlugin%' ORDER BY `period` ASC , `date1` ASC, `date2` ASC, `value` ASC"); $expectedDoneFlags = [ ['period' => Day::PERIOD_ID, 'date1' => '2007-04-04', 'date2' => '2007-04-04', 'value' => ArchiveWriter::DONE_PARTIAL], ['period' => Day::PERIOD_ID, 'date1' => '2007-04-05', 'date2' => '2007-04-05', 'value' => ArchiveWriter::DONE_OK], ['period' => Week::PERIOD_ID, 'date1' => '2007-04-02', 'date2' => '2007-04-08', 'value' => ArchiveWriter::DONE_OK], ['period' => Week::PERIOD_ID, 'date1' => '2007-04-02', 'date2' => '2007-04-08', 'value' => ArchiveWriter::DONE_PARTIAL], ['period' => Month::PERIOD_ID, 'date1' => '2007-04-01', 'date2' => '2007-04-30', 'value' => ArchiveWriter::DONE_OK], ['period' => Month::PERIOD_ID, 'date1' => '2007-04-01', 'date2' => '2007-04-30', 'value' => ArchiveWriter::DONE_PARTIAL], ]; self::assertEquals($expectedDoneFlags, $doneFlags); } /** * @depends testArchivePhpCronWithSingleReportRearchive */ public function testArchivePhpCronArchivesFullRanges() { self::$fixture->getTestEnvironment()->overrideConfig('General', 'enable_browser_archiving_triggering', 0); self::$fixture->getTestEnvironment()->overrideConfig('General', 'archiving_range_force_on_browser_request', 0); self::$fixture->getTestEnvironment()->overrideConfig('General', 'archiving_custom_ranges', ['2012-08-09,2012-08-13']); self::$fixture->getTestEnvironment()->save(); Config::getInstance()->General['enable_browser_archiving_triggering'] = 0; Config::getInstance()->General['archiving_range_force_on_browser_request'] = 0; Config::getInstance()->General['archiving_custom_ranges'][] = ''; Rules::setBrowserTriggerArchiving(false); $output = $this->runArchivePhpCron(['--force-periods' => 'range', '--force-idsites' => 1]); $expectedInvalidations = []; $invalidationEntries = $this->getInvalidatedArchiveTableEntries(); $invalidationEntries = array_filter($invalidationEntries, function ($entry) { return $entry['period'] == 5; }); $this->assertEquals($expectedInvalidations, $invalidationEntries); $this->runApiTests( array( 'VisitsSummary.get', 'Actions.get', 'DevicesDetection.getType'), array('idSite' => '1', 'date' => '2012-08-09,2012-08-13', 'periods' => array('range'), 'testSuffix' => '_range_archive') ); } /** * @depends testArchivePhpCronArchivesFullRanges */ public function testRangeBrowserArchivingWorksWhenInvalidatedByVisit() { self::$fixture->getTestEnvironment()->overrideConfig('General', 'enable_browser_archiving_triggering', 0); // remove the value, since the default is to always force archiving on browser request self::$fixture->getTestEnvironment()->removeOverriddenConfig('General', 'archiving_range_force_on_browser_request'); self::$fixture->getTestEnvironment()->save(); Config::getInstance()->General['enable_browser_archiving_triggering'] = 0; unset(Config::getInstance()->General['archiving_range_force_on_browser_request']); $table = ArchiveTableCreator::getNumericTable(Date::factory('2012-08-09'), true); $blobTable = ArchiveTableCreator::getBlobTable(Date::factory('2012-08-09'), true); // remove existing range archives $idArchives = Db::fetchAll("SELECT DISTINCT idarchive FROM `$table` WHERE period = 5"); $idArchives = array_column($idArchives, 'idarchive'); $model = new Model(); $model->deleteArchiveIds($table, $blobTable, $idArchives); // process archives once $this->runApiTests( array( 'VisitsSummary.get', 'Actions.get', 'DevicesDetection.getType'), array('idSite' => '1', 'date' => '2012-08-09,2012-08-13', 'periods' => array('range'), 'testSuffix' => '_range_archive_before_invalidate') ); $this->trackVisitInPast($ip = '124.56.23.23'); $cronArchive = new CronArchive(); $cronArchive->init(); $cronArchive->invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain(1); // process day archive so range archive will process below (if we don't, then we'll skip processing the range archive, // since it wouldn't trigger the child day archive. then later the day archive would be updated, and not the range, resulting // in inaccurate data) Rules::setBrowserTriggerArchiving(true); \Piwik\Plugins\VisitsSummary\API::getInstance()->get(1, 'day', '2012-08-09'); Rules::setBrowserTriggerArchiving(false); $rangeArchivesInvalid = Db::fetchAll("SELECT idarchive, name, date1, date2 FROM " . $table . " WHERE period = 5 and name LIKE 'done.%' and value = " . ArchiveWriter::DONE_INVALIDATED); $this->assertNotEmpty($rangeArchivesInvalid); // cron archive not run, but ranges should still be rearchived $this->runApiTests( array( 'VisitsSummary.get', 'Actions.get', 'DevicesDetection.getType'), array('idSite' => '1', 'date' => '2012-08-09,2012-08-13', 'periods' => array('range'), 'testSuffix' => '_range_archive_rearchived') ); $archivePurger = StaticContainer::get(ArchivePurger::class); $archivePurger->purgeInvalidatedArchivesFrom(Date::factory('2020-08-09')); $rangeArchivesStillInvalid = Db::fetchAll("SELECT idarchive, name, date1, date2 FROM " . $table . " WHERE period = 5 and name = 'done.%' and value = " . ArchiveWriter::DONE_INVALIDATED); $this->assertEmpty($rangeArchivesStillInvalid); } public function testArchivePhpScriptDoesNotFailWhenCommandHelpRequested() { $output = $this->runArchivePhpCron(array('--help' => null), PIWIK_INCLUDE_PATH . '/misc/cron/archive.php'); $this->assertRegExp('/Usage:\s*core:archive/', $output); self::assertStringNotContainsString("Starting Piwik reports archiving...", $output); } public function testArchivePhpCronWithTimeLimit() { self::$fixture->getTestEnvironment()->overrideConfig('General', 'enable_browser_archiving_triggering', 0); self::$fixture->getTestEnvironment()->overrideConfig('General', 'browser_archiving_disabled_enforce', 1); self::$fixture->getTestEnvironment()->save(); Config::getInstance()->General['enable_browser_archiving_triggering'] = 0; Config::getInstance()->General['browser_archiving_disabled_enforce'] = 1; Rules::setBrowserTriggerArchiving(false); // track a visit so archiving will go through $tracker = Fixture::getTracker(1, '2005-09-04'); $tracker->setUrl('http://example.com/test/url2'); Fixture::checkResponse($tracker->doTrackPageView('jkl')); $output = $this->runArchivePhpCron(['-vvv --stop-processing-after=1' => null]); self::assertStringContainsString('Maximum time limit per execution has been reached.', $output); // Ensure work has stopped and some invalidations are left over self::assertNotEmpty($this->getInvalidatedArchiveTableEntries()); } private function runArchivePhpCron($options = array(), $archivePhpScript = false) { $archivePhpScript = $archivePhpScript ?: PIWIK_INCLUDE_PATH . '/tests/PHPUnit/proxy/archive.php'; $urlToProxy = Fixture::getRootUrl() . 'tests/PHPUnit/proxy/index.php'; // create the command $cmd = "php \"$archivePhpScript\" --url=\"$urlToProxy\""; foreach ($options as $name => $value) { $cmd .= " $name"; if ($value !== null) { $cmd .= "=" . escapeshellarg($value); } } $cmd .= " 2>&1"; // run the command exec($cmd, $output, $result); $output = implode("\n", $output); if ($result !== 0 || strpos($output, "ERROR") || strpos($output, "Error")) { $this->fail("archive cron failed (result = $result): " . $output . "\n\ncommand used: $cmd"); } return $output; } public static function provideContainerConfigBeforeClass() { Date::$now = strtotime('2015-01-01 00:00:00'); return array( LoggerInterface::class => \Piwik\DI::get(Logger::class), // for some reason, w/o real translations archiving segments in CronArchive fails. the data returned by CliMulti // is a translation token, and nothing else. 'Piwik\Translation\Translator' => function (Container $c) { return new \Piwik\Translation\Translator($c->get('Piwik\Translation\Loader\LoaderInterface')); }, 'Tests.log.allowAllHandlers' => true, CronArchive\SegmentArchiving::class => \Piwik\DI::autowire() // Oldest reports are for 2012, so ensure segments are processed for that year ->constructorParameter('beginningOfTimeLastNInYears', date('Y') - 2012), ); } public static function getPathToTestDirectory() { return dirname(__FILE__); } private function getInvalidatedArchiveTableEntries() { return Db::fetchAll("SELECT idinvalidation, idarchive, idsite, date1, date2, period, name, status FROM " . Common::prefixTable('archive_invalidations')); } private static function undoForceCurlCliMulti() { $testVars = new TestingEnvironmentVariables(); $testVars->forceCliMultiViaCurl = 0; $testVars->save(); } private static function forceCurlCliMulti() { $testVars = new TestingEnvironmentVariables(); $testVars->forceCliMultiViaCurl = 1; $testVars->save(); } } ArchiveCronTest::$fixture = new ManySitesImportedLogs(); ArchiveCronTest::$fixture->addSegments = true;