MediaWiki r13088 - Code Review

Jump to: navigation, search
Repository:MediaWiki
Revision:< r13087‎ | r13088 (on ViewVC)‎ | r13089 >
Date:01:56, 24 February 2006
Author:timstarling
Status:old
Tags:
Comment:
Added job table, for deferred processing of jobs. The immediate application is to complete the link table refresh operation when templates are changed.
Modified paths:

Diff [purge]

Index: trunk/phase3/maintenance/archives/patch-job.sql
——@@ -0,0 +1,20 @@
 2+
 3+-- Jobs performed by parallel apache threads or a command-line daemon
 4+CREATE TABLE /*$wgDBprefix*/job (
 5+ job_id int(9) unsigned NOT NULL auto_increment,
 6+
 7+ -- Command name, currently only refreshLinks is defined
 8+ job_cmd varchar(255) NOT NULL default '',
 9+
 10+ -- Namespace and title to act on
 11+ -- Should be 0 and '' if the command does not operate on a title
 12+ job_namespace int NOT NULL,
 13+ job_title varchar(255) binary NOT NULL,
 14+
 15+ -- Any other parameters to the command
 16+ -- Presently unused, format undefined
 17+ job_params blob NOT NULL default '',
 18+
 19+ PRIMARY KEY job_id (job_id),
 20+ KEY (job_cmd, job_namespace, job_title)
 21+) TYPE=InnoDB;
Property changes on: trunk/phase3/maintenance/archives/patch-job.sql
___________________________________________________________________
Added: svn:eol-style
122 + native
Added: svn:keywords
223 + Author Date Id Revision
Index: trunk/phase3/maintenance/runJobs.php
——@@ -0,0 +1,13 @@
 2+<?php
 3+
 4+require_once( 'commandLine.inc' );
 5+require_once( "$IP/includes/JobQueue.php" );
 6+
 7+while ( false != ($job = Job::pop()) ) {
 8+ wfWaitForSlaves( 5 );
 9+ print $job->toString() . "\n";
 10+ if ( !$job->run() ) {
 11+ print "Error: {$job->error}\n";
 12+ }
 13+}
 14+
Property changes on: trunk/phase3/maintenance/runJobs.php
___________________________________________________________________
Added: svn:eol-style
115 + native
Added: svn:keywords
216 + Author Date Id Revision
Index: trunk/phase3/maintenance/mysql5/tables.sql
——@@ -914,3 +914,23 @@
915915
916916 INDEX (tb_page)
917917 ) TYPE=InnoDB, DEFAULT CHARSET=utf8;
 918+
 919+-- Jobs performed by parallel apache threads or a command-line daemon
 920+CREATE TABLE /*$wgDBprefix*/job (
 921+ job_id int(9) unsigned NOT NULL auto_increment,
 922+
 923+ -- Command name, currently only refreshLinks is defined
 924+ job_cmd varchar(255) NOT NULL default '',
 925+
 926+ -- Namespace and title to act on
 927+ -- Should be 0 and '' if the command does not operate on a title
 928+ job_namespace int NOT NULL,
 929+ job_title varchar(255) binary NOT NULL,
 930+
 931+ -- Any other parameters to the command
 932+ -- Presently unused, format undefined
 933+ job_params blob NOT NULL default '',
 934+
 935+ PRIMARY KEY job_id (job_id),
 936+ KEY (job_cmd, job_namespace, job_title)
 937+) TYPE=InnoDB, DEFAULT CHARSET=utf8;
Index: trunk/phase3/maintenance/updaters.inc
——@@ -27,6 +27,7 @@
2828 array( 'transcache', 'patch-transcache.sql' ),
2929 array( 'trackbacks', 'patch-trackbacks.sql' ),
3030 array( 'externallinks', 'patch-externallinks.sql' ),
 31+ array( 'job', 'patch-job.sql' ),
3132 );
3233
3334 $wgNewFields = array(
Index: trunk/phase3/maintenance/tables.sql
——@@ -902,3 +902,24 @@
903903
904904 INDEX (tb_page)
905905 ) TYPE=InnoDB;
 906+
 907+
 908+-- Jobs performed by parallel apache threads or a command-line daemon
 909+CREATE TABLE /*$wgDBprefix*/job (
 910+ job_id int(9) unsigned NOT NULL auto_increment,
 911+
 912+ -- Command name, currently only refreshLinks is defined
 913+ job_cmd varchar(255) NOT NULL default '',
 914+
 915+ -- Namespace and title to act on
 916+ -- Should be 0 and '' if the command does not operate on a title
 917+ job_namespace int NOT NULL,
 918+ job_title varchar(255) binary NOT NULL,
 919+
 920+ -- Any other parameters to the command
 921+ -- Presently unused, format undefined
 922+ job_params blob NOT NULL default '',
 923+
 924+ PRIMARY KEY job_id (job_id),
 925+ KEY (job_cmd, job_namespace, job_title)
 926+) TYPE=InnoDB;
Index: trunk/phase3/RELEASE-NOTES
——@@ -65,7 +65,10 @@
6666 * Respect database prefix in dumpHTML.inc
6767 * Removed read-only check from Database::query()
6868 * Added externallinks table, to track links to arbitrary URLs
 69+* Added job table, for deferred processing of jobs. The immediate application is
 70+ to complete the link table refresh operation when templates are changed.
6971
 72+
7073 Documentation:
7174 * (bug 3306) Document $wgLocalTZoffset
7275
Index: trunk/phase3/includes/JobQueue.php
——@@ -0,0 +1,182 @@
 2+<?php
 3+
 4+if ( !defined( 'MEDIAWIKI' ) ) {
 5+ die( "This file is part of MediaWiki, it is not a valid entry point\n" );
 6+}
 7+
 8+class Job {
 9+ var $command,
 10+ $title,
 11+ $params,
 12+ $removeDuplicates,
 13+ $error;
 14+
 15+ /*-------------------------------------------------------------------------
 16+ * Static functions
 17+ *------------------------------------------------------------------------*/
 18+ /**
 19+ * Add an array of refreshLinks jobs to the queue
 20+ * @param array $titles Array of title objects.
 21+ * @static
 22+ */
 23+ function queueLinksJobs( $titles ) {
 24+ $fname = 'Job::queueLinksJobs';
 25+ wfProfileIn( $fname );
 26+ foreach ( $titles as $title ) {
 27+ $job = new Job( 'refreshLinks', $title );
 28+ $job->insert();
 29+ }
 30+ wfProfileOut( $fname );
 31+ }
 32+
 33+ /**
 34+ * Pop a job off the front of the queue
 35+ * @static
 36+ * @return Job or false if there's no jobs
 37+ */
 38+ function pop() {
 39+ $fname = 'Job::pop';
 40+ wfProfileIn( $fname );
 41+
 42+ // First check to see if there are any jobs in the slave DB
 43+ $dbr =& wfGetDB( DB_SLAVE );
 44+ $id = $dbr->selectField( 'job', 'job_id', '', $fname, array( 'LIMIT' => 1 ) );
 45+ if ( $id === false ) {
 46+ wfProfileOut( $fname );
 47+ return false;
 48+ }
 49+
 50+ // Pop an item off the front of the queue
 51+ // Method due to Domas, may not work on all DBMSes
 52+ $dbw =& wfGetDB( DB_MASTER );
 53+ $jobTable = $dbw->tableName( 'job' );
 54+ $dbw->query( "DELETE FROM $jobTable WHERE " .
 55+ '(job_cmd = @job_cmd := job_cmd) AND ' .
 56+ '(job_namespace = @job_namespace := job_namespace) AND ' .
 57+ '(job_title = @job_title := job_title) AND ' .
 58+ '(job_params = @job_params := job_params) ' .
 59+ 'ORDER BY job_id LIMIT 1', $fname );
 60+ $affected = $dbw->affectedRows();
 61+ // Commit now before 100 other threads pile up behind us
 62+ $dbw->immediateCommit();
 63+ if ( !$affected ) {
 64+ wfProfileOut( $fname );
 65+ return false;
 66+ }
 67+
 68+ $res = $dbw->query( "SELECT @job_cmd, @job_namespace, @job_title, @job_params", $fname );
 69+ $row = $dbw->fetchRow( $res );
 70+ if ( !$row ) {
 71+ wfProfileOut( $fname );
 72+ return false;
 73+ }
 74+
 75+ $command = $row['@job_cmd'];
 76+ $namespace = $row['@job_namespace'];
 77+ $dbkey = $row['@job_title'];
 78+ $title = Title::makeTitleSafe( $namespace, $dbkey );
 79+ $params = $row['@job_params'];
 80+ $job = new Job( $command, $title, $params );
 81+ wfProfileOut( $fname );
 82+ return $job;
 83+ }
 84+
 85+ /*-------------------------------------------------------------------------
 86+ * Non-static functions
 87+ *------------------------------------------------------------------------*/
 88+
 89+ function Job( $command, $title, $params = '' ) {
 90+ $this->command = $command;
 91+ $this->title = $title;
 92+ $this->params = $params;
 93+
 94+ // A bit of premature generalisation
 95+ // Oh well, the whole class is premature generalisation really
 96+ $this->removeDuplicates = true;
 97+ }
 98+
 99+ function insert() {
 100+ $fname = 'Job::insert';
 101+
 102+ $fields = array(
 103+ 'job_cmd' => $this->command,
 104+ 'job_namespace' => $this->title->getNamespace(),
 105+ 'job_title' => $this->title->getDBkey(),
 106+ 'job_params' => $this->params
 107+ );
 108+
 109+ $dbw =& wfGetDB( DB_MASTER );
 110+
 111+ if ( $this->removeDuplicates ) {
 112+ $dbw->delete( 'job', $fields, $fname );
 113+ }
 114+ $fields['job_id'] = $dbw->nextSequenceValue( 'job_job_id_seq' );
 115+ $dbw->insert( 'job', $fields, $fname );
 116+ }
 117+
 118+ /**
 119+ * Run the job
 120+ * @return boolean success
 121+ */
 122+ function run() {
 123+ $fname = 'Job::run';
 124+ wfProfileIn( $fname );
 125+ switch ( $this->command ) {
 126+ case 'refreshLinks':
 127+ $retval = $this->refreshLinks();
 128+ break;
 129+ default:
 130+ $this->error = "Invalid job type {$this->command}, ignoring";
 131+ wfDebug( $this->error . "\n" );
 132+ $retval = false;
 133+ }
 134+ wfProfileOut( $fname );
 135+ return $retval;
 136+ }
 137+
 138+ /**
 139+ * Run a refreshLinks job
 140+ * @return boolean success
 141+ */
 142+ function refreshLinks() {
 143+ global $wgParser;
 144+
 145+ $dbw =& wfGetDB( DB_MASTER );
 146+
 147+ $linkCache =& LinkCache::singleton();
 148+ $linkCache->clear();
 149+
 150+ if ( is_null( $this->title ) ) {
 151+ $this->error = "refreshLinks: Invalid title";
 152+ return false;
 153+ }
 154+
 155+ $revision = Revision::newFromTitle( $this->title );
 156+ if ( !$revision ) {
 157+ $this->error = 'refreshLinks: Article not found "' . $this->title->getPrefixedDBkey() . '"';
 158+ return false;
 159+ }
 160+
 161+ $options = new ParserOptions;
 162+ $parserOutput = $wgParser->parse( $revision->getText(), $this->title, $options, true, true, $revision->getId() );
 163+ $update = new LinksUpdate( $this->title, $parserOutput );
 164+ $update->doUpdate();
 165+ return true;
 166+ }
 167+
 168+ function toString() {
 169+ if ( is_object( $this->title ) ) {
 170+ $s = "{$this->command} " . $this->title->getPrefixedDBkey();
 171+ if ( $this->params !== '' ) {
 172+ $s .= ', ' . $this->params;
 173+ }
 174+ return $s;
 175+ } else {
 176+ return "{$this->command} {$this->params}";
 177+ }
 178+ }
 179+
 180+ function getLastError() {
 181+ return $this->error;
 182+ }
 183+}
Property changes on: trunk/phase3/includes/JobQueue.php
___________________________________________________________________
Added: svn:eol-style
1184 + native
Added: svn:keywords
2185 + Author Date Id Revision
Index: trunk/phase3/includes/Wiki.php
——@@ -249,6 +249,7 @@
250250 function finalCleanup ( &$deferredUpdates, &$loadBalancer, &$output ) {
251251 wfProfileIn( 'MediaWiki::finalCleanup' );
252252 $this->doUpdates( $deferredUpdates );
 253+ $this->doJobs();
253254 $loadBalancer->saveMasterPos();
254255 # Now commit any transactions, so that unreported errors after output() don't roll back the whole thing
255256 $loadBalancer->commitAll();
——@@ -268,6 +269,38 @@
269270 }
270271 wfProfileOut( 'MediaWiki::doUpdates' );
271272 }
 273+
 274+ /**
 275+ * Do a job from the job queue
 276+ */
 277+ function doJobs() {
 278+ global $wgJobLogFile, $wgJobRunRate;
 279+
 280+ if ( $wgJobRunRate <= 0 ) {
 281+ return;
 282+ }
 283+ if ( $wgJobRunRate < 1 ) {
 284+ $max = mt_getrandmax();
 285+ if ( mt_rand( 0, $max ) < $max * $wgJobRunRate ) {
 286+ return;
 287+ }
 288+ $n = 1;
 289+ } else {
 290+ $n = intval( $wgJobRunRate );
 291+ }
 292+
 293+ require_once( 'JobQueue.php' );
 294+
 295+ while ( $n-- && false != ($job = Job::pop())) {
 296+ $output = $job->toString() . "\n";
 297+ if ( !$job->run() ) {
 298+ $output .= "Error: " . $job->getLastError() . "\n";
 299+ }
 300+ if ( $wgJobLogFile ) {
 301+ error_log( $output, 3, $wgJobLogFile );
 302+ }
 303+ }
 304+ }
272305
273306 /**
274307 * Ends this task peacefully
Index: trunk/phase3/includes/LinksUpdate.php
——@@ -15,7 +15,6 @@
1616 */
1717 var $mId, # Page ID of the article linked from
1818 $mTitle, # Title object of the article linked from
19 - $mParserOutput, # Parser output containing the links to be inserted into the database
2019 $mLinks, # Map of title strings to IDs for the links in the document
2120 $mImages, # DB keys of the images used, in the array key only
2221 $mTemplates, # Map of title strings to IDs for the template references, including broken ones
——@@ -47,14 +46,12 @@
4847 }
4948 $this->mTitle = $title;
5049 $this->mId = $title->getArticleID();
51 - $this->mParserOutput = $parserOutput;
5250
53 - // Shortcut aliases
54 - $this->mLinks =& $this->mParserOutput->getLinks();
55 - $this->mImages =& $this->mParserOutput->getImages();
56 - $this->mTemplates =& $this->mParserOutput->getTemplates();
57 - $this->mExternals =& $this->mParserOutput->getExternalLinks();
58 - $this->mCategories =& $this->mParserOutput->getCategories();
 51+ $this->mLinks = $parserOutput->getLinks();
 52+ $this->mImages = $parserOutput->getImages();
 53+ $this->mTemplates = $parserOutput->getTemplates();
 54+ $this->mExternals = $parserOutput->getExternalLinks();
 55+ $this->mCategories = $parserOutput->getCategories();
5956
6057 }
6158
——@@ -79,11 +76,6 @@
8077 $this->incrTableUpdate( 'pagelinks', 'pl', $this->getLinkDeletions( $existing ),
8178 $this->getLinkInsertions( $existing ) );
8279
83 - # Template links
84 - $existing = $this->getExistingTemplates();
85 - $this->incrTableUpdate( 'templatelinks', 'tl', $this->getTemplateDeletions( $existing ),
86 - $this->getTemplateInsertions( $existing ) );
87 -
8880 # Image links
8981 $existing = $this->getExistingImages();
9082 $this->incrTableUpdate( 'imagelinks', 'il', $this->getImageDeletions( $existing ),
——@@ -93,7 +85,19 @@
9486 $existing = $this->getExistingExternals();
9587 $this->incrTableUpdate( 'externallinks', 'el', $this->getExternalDeletions( $existing ),
9688 $this->getExternalInsertions( $existing ) );
97 -
 89+
 90+ # Template links
 91+ $existing = $this->getExistingTemplates();
 92+ $this->incrTableUpdate( 'templatelinks', 'tl', $this->getTemplateDeletions( $existing ),
 93+ $this->getTemplateInsertions( $existing ) );
 94+
 95+ # Refresh links of all pages including this page
 96+ $tlto = $this->mTitle->getTemplateLinksTo();
 97+ if ( count( $tlto ) ) {
 98+ require_once( 'JobQueue.php' );
 99+ Job::queueLinksJobs( $tlto );
 100+ }
 101+
98102 # Category links
99103 $existing = $this->getExistingCategories();
100104 $this->incrTableUpdate( 'categorylinks', 'cl', $this->getCategoryDeletions( $existing ),
——@@ -117,9 +121,17 @@
118122 $fname = 'LinksUpdate::doDumbUpdate';
119123 wfProfileIn( $fname );
120124
 125+ # Refresh category pages
121126 $existing = $this->getExistingCategories();
122127 $categoryUpdates = array_diff_assoc( $existing, $this->mCategories ) + array_diff_assoc( $this->mCategories, $existing );
123128
 129+ # Refresh links of all pages including this page
 130+ $tlto = $this->mTitle->getTemplateLinksTo();
 131+ if ( count( $tlto ) ) {
 132+ require_once( 'JobQueue.php' );
 133+ Job::queueLinksJobs( $tlto );
 134+ }
 135+
124136 $this->dumbTableUpdate( 'pagelinks', $this->getLinkInsertions(), 'pl_from' );
125137 $this->dumbTableUpdate( 'imagelinks', $this->getImageInsertions(), 'il_from' );
126138 $this->dumbTableUpdate( 'categorylinks', $this->getCategoryInsertions(), 'cl_from' );
Index: trunk/phase3/includes/DefaultSettings.php
——@@ -1865,4 +1865,18 @@
18661866 */
18671867 $wgAllowCategorizedRecentChanges = false ;
18681868
 1869+/**
 1870+ * Number of jobs to perform per request. May be less than one in which case
 1871+ * jobs are performed probabalistically. If this is zero, jobs will not be done
 1872+ * during ordinary apache requests. In this case, maintenance/doJobs.php should
 1873+ * be run periodically.
 1874+ */
 1875+$wgJobRunRate = 1;
 1876+
 1877+/**
 1878+ * Log file for job execution
 1879+ */
 1880+$wgJobLogFile = false;
 1881+
 1882+
18691883 ?>

Status & tagging log

  • 01:58, 13 October 2010 ^demon (talk | contribs) changed the status of r13088 [removed: new added: old]
Retrieved from "http://www.mediawiki.org/wiki/Special:Code/MediaWiki/13088"