| Index: trunk/phase3/maintenance/archives/patch-job.sql |
| — | — | @@ -0,0 +1,20 @@ |
| | 2 | + |
| | 3 | +-- Jobs performed by parallel apache threads or a command-line daemon |
| | 4 | +CREATE TABLE /*$wgDBprefix*/job ( |
| | 5 | + job_id int(9) unsigned NOT NULL auto_increment, |
| | 6 | + |
| | 7 | + -- Command name, currently only refreshLinks is defined |
| | 8 | + job_cmd varchar(255) NOT NULL default '', |
| | 9 | + |
| | 10 | + -- Namespace and title to act on |
| | 11 | + -- Should be 0 and '' if the command does not operate on a title |
| | 12 | + job_namespace int NOT NULL, |
| | 13 | + job_title varchar(255) binary NOT NULL, |
| | 14 | + |
| | 15 | + -- Any other parameters to the command |
| | 16 | + -- Presently unused, format undefined |
| | 17 | + job_params blob NOT NULL default '', |
| | 18 | + |
| | 19 | + PRIMARY KEY job_id (job_id), |
| | 20 | + KEY (job_cmd, job_namespace, job_title) |
| | 21 | +) TYPE=InnoDB; |
| Property changes on: trunk/phase3/maintenance/archives/patch-job.sql |
| ___________________________________________________________________ |
| Added: svn:eol-style |
| 1 | 22 | + native |
| Added: svn:keywords |
| 2 | 23 | + Author Date Id Revision |
| Index: trunk/phase3/maintenance/runJobs.php |
| — | — | @@ -0,0 +1,13 @@ |
| | 2 | +<?php |
| | 3 | + |
| | 4 | +require_once( 'commandLine.inc' ); |
| | 5 | +require_once( "$IP/includes/JobQueue.php" ); |
| | 6 | + |
| | 7 | +while ( false != ($job = Job::pop()) ) { |
| | 8 | + wfWaitForSlaves( 5 ); |
| | 9 | + print $job->toString() . "\n"; |
| | 10 | + if ( !$job->run() ) { |
| | 11 | + print "Error: {$job->error}\n"; |
| | 12 | + } |
| | 13 | +} |
| | 14 | + |
| Property changes on: trunk/phase3/maintenance/runJobs.php |
| ___________________________________________________________________ |
| Added: svn:eol-style |
| 1 | 15 | + native |
| Added: svn:keywords |
| 2 | 16 | + Author Date Id Revision |
| Index: trunk/phase3/maintenance/mysql5/tables.sql |
| — | — | @@ -914,3 +914,23 @@ |
| 915 | 915 | |
| 916 | 916 | INDEX (tb_page) |
| 917 | 917 | ) TYPE=InnoDB, DEFAULT CHARSET=utf8; |
| | 918 | + |
| | 919 | +-- Jobs performed by parallel apache threads or a command-line daemon |
| | 920 | +CREATE TABLE /*$wgDBprefix*/job ( |
| | 921 | + job_id int(9) unsigned NOT NULL auto_increment, |
| | 922 | + |
| | 923 | + -- Command name, currently only refreshLinks is defined |
| | 924 | + job_cmd varchar(255) NOT NULL default '', |
| | 925 | + |
| | 926 | + -- Namespace and title to act on |
| | 927 | + -- Should be 0 and '' if the command does not operate on a title |
| | 928 | + job_namespace int NOT NULL, |
| | 929 | + job_title varchar(255) binary NOT NULL, |
| | 930 | + |
| | 931 | + -- Any other parameters to the command |
| | 932 | + -- Presently unused, format undefined |
| | 933 | + job_params blob NOT NULL default '', |
| | 934 | + |
| | 935 | + PRIMARY KEY job_id (job_id), |
| | 936 | + KEY (job_cmd, job_namespace, job_title) |
| | 937 | +) TYPE=InnoDB, DEFAULT CHARSET=utf8; |
| Index: trunk/phase3/maintenance/updaters.inc |
| — | — | @@ -27,6 +27,7 @@ |
| 28 | 28 | array( 'transcache', 'patch-transcache.sql' ), |
| 29 | 29 | array( 'trackbacks', 'patch-trackbacks.sql' ), |
| 30 | 30 | array( 'externallinks', 'patch-externallinks.sql' ), |
| | 31 | + array( 'job', 'patch-job.sql' ), |
| 31 | 32 | ); |
| 32 | 33 | |
| 33 | 34 | $wgNewFields = array( |
| Index: trunk/phase3/maintenance/tables.sql |
| — | — | @@ -902,3 +902,24 @@ |
| 903 | 903 | |
| 904 | 904 | INDEX (tb_page) |
| 905 | 905 | ) TYPE=InnoDB; |
| | 906 | + |
| | 907 | + |
| | 908 | +-- Jobs performed by parallel apache threads or a command-line daemon |
| | 909 | +CREATE TABLE /*$wgDBprefix*/job ( |
| | 910 | + job_id int(9) unsigned NOT NULL auto_increment, |
| | 911 | + |
| | 912 | + -- Command name, currently only refreshLinks is defined |
| | 913 | + job_cmd varchar(255) NOT NULL default '', |
| | 914 | + |
| | 915 | + -- Namespace and title to act on |
| | 916 | + -- Should be 0 and '' if the command does not operate on a title |
| | 917 | + job_namespace int NOT NULL, |
| | 918 | + job_title varchar(255) binary NOT NULL, |
| | 919 | + |
| | 920 | + -- Any other parameters to the command |
| | 921 | + -- Presently unused, format undefined |
| | 922 | + job_params blob NOT NULL default '', |
| | 923 | + |
| | 924 | + PRIMARY KEY job_id (job_id), |
| | 925 | + KEY (job_cmd, job_namespace, job_title) |
| | 926 | +) TYPE=InnoDB; |
| Index: trunk/phase3/RELEASE-NOTES |
| — | — | @@ -65,7 +65,10 @@ |
| 66 | 66 | * Respect database prefix in dumpHTML.inc |
| 67 | 67 | * Removed read-only check from Database::query() |
| 68 | 68 | * Added externallinks table, to track links to arbitrary URLs |
| | 69 | +* Added job table, for deferred processing of jobs. The immediate application is |
| | 70 | + to complete the link table refresh operation when templates are changed. |
| 69 | 71 | |
| | 72 | + |
| 70 | 73 | Documentation: |
| 71 | 74 | * (bug 3306) Document $wgLocalTZoffset |
| 72 | 75 | |
| Index: trunk/phase3/includes/JobQueue.php |
| — | — | @@ -0,0 +1,182 @@ |
| | 2 | +<?php |
| | 3 | + |
| | 4 | +if ( !defined( 'MEDIAWIKI' ) ) { |
| | 5 | + die( "This file is part of MediaWiki, it is not a valid entry point\n" ); |
| | 6 | +} |
| | 7 | + |
| | 8 | +class Job { |
| | 9 | + var $command, |
| | 10 | + $title, |
| | 11 | + $params, |
| | 12 | + $removeDuplicates, |
| | 13 | + $error; |
| | 14 | + |
| | 15 | + /*------------------------------------------------------------------------- |
| | 16 | + * Static functions |
| | 17 | + *------------------------------------------------------------------------*/ |
| | 18 | + /** |
| | 19 | + * Add an array of refreshLinks jobs to the queue |
| | 20 | + * @param array $titles Array of title objects. |
| | 21 | + * @static |
| | 22 | + */ |
| | 23 | + function queueLinksJobs( $titles ) { |
| | 24 | + $fname = 'Job::queueLinksJobs'; |
| | 25 | + wfProfileIn( $fname ); |
| | 26 | + foreach ( $titles as $title ) { |
| | 27 | + $job = new Job( 'refreshLinks', $title ); |
| | 28 | + $job->insert(); |
| | 29 | + } |
| | 30 | + wfProfileOut( $fname ); |
| | 31 | + } |
| | 32 | + |
| | 33 | + /** |
| | 34 | + * Pop a job off the front of the queue |
| | 35 | + * @static |
| | 36 | + * @return Job or false if there's no jobs |
| | 37 | + */ |
| | 38 | + function pop() { |
| | 39 | + $fname = 'Job::pop'; |
| | 40 | + wfProfileIn( $fname ); |
| | 41 | + |
| | 42 | + // First check to see if there are any jobs in the slave DB |
| | 43 | + $dbr =& wfGetDB( DB_SLAVE ); |
| | 44 | + $id = $dbr->selectField( 'job', 'job_id', '', $fname, array( 'LIMIT' => 1 ) ); |
| | 45 | + if ( $id === false ) { |
| | 46 | + wfProfileOut( $fname ); |
| | 47 | + return false; |
| | 48 | + } |
| | 49 | + |
| | 50 | + // Pop an item off the front of the queue |
| | 51 | + // Method due to Domas, may not work on all DBMSes |
| | 52 | + $dbw =& wfGetDB( DB_MASTER ); |
| | 53 | + $jobTable = $dbw->tableName( 'job' ); |
| | 54 | + $dbw->query( "DELETE FROM $jobTable WHERE " . |
| | 55 | + '(job_cmd = @job_cmd := job_cmd) AND ' . |
| | 56 | + '(job_namespace = @job_namespace := job_namespace) AND ' . |
| | 57 | + '(job_title = @job_title := job_title) AND ' . |
| | 58 | + '(job_params = @job_params := job_params) ' . |
| | 59 | + 'ORDER BY job_id LIMIT 1', $fname ); |
| | 60 | + $affected = $dbw->affectedRows(); |
| | 61 | + // Commit now before 100 other threads pile up behind us |
| | 62 | + $dbw->immediateCommit(); |
| | 63 | + if ( !$affected ) { |
| | 64 | + wfProfileOut( $fname ); |
| | 65 | + return false; |
| | 66 | + } |
| | 67 | + |
| | 68 | + $res = $dbw->query( "SELECT @job_cmd, @job_namespace, @job_title, @job_params", $fname ); |
| | 69 | + $row = $dbw->fetchRow( $res ); |
| | 70 | + if ( !$row ) { |
| | 71 | + wfProfileOut( $fname ); |
| | 72 | + return false; |
| | 73 | + } |
| | 74 | + |
| | 75 | + $command = $row['@job_cmd']; |
| | 76 | + $namespace = $row['@job_namespace']; |
| | 77 | + $dbkey = $row['@job_title']; |
| | 78 | + $title = Title::makeTitleSafe( $namespace, $dbkey ); |
| | 79 | + $params = $row['@job_params']; |
| | 80 | + $job = new Job( $command, $title, $params ); |
| | 81 | + wfProfileOut( $fname ); |
| | 82 | + return $job; |
| | 83 | + } |
| | 84 | + |
| | 85 | + /*------------------------------------------------------------------------- |
| | 86 | + * Non-static functions |
| | 87 | + *------------------------------------------------------------------------*/ |
| | 88 | + |
| | 89 | + function Job( $command, $title, $params = '' ) { |
| | 90 | + $this->command = $command; |
| | 91 | + $this->title = $title; |
| | 92 | + $this->params = $params; |
| | 93 | + |
| | 94 | + // A bit of premature generalisation |
| | 95 | + // Oh well, the whole class is premature generalisation really |
| | 96 | + $this->removeDuplicates = true; |
| | 97 | + } |
| | 98 | + |
| | 99 | + function insert() { |
| | 100 | + $fname = 'Job::insert'; |
| | 101 | + |
| | 102 | + $fields = array( |
| | 103 | + 'job_cmd' => $this->command, |
| | 104 | + 'job_namespace' => $this->title->getNamespace(), |
| | 105 | + 'job_title' => $this->title->getDBkey(), |
| | 106 | + 'job_params' => $this->params |
| | 107 | + ); |
| | 108 | + |
| | 109 | + $dbw =& wfGetDB( DB_MASTER ); |
| | 110 | + |
| | 111 | + if ( $this->removeDuplicates ) { |
| | 112 | + $dbw->delete( 'job', $fields, $fname ); |
| | 113 | + } |
| | 114 | + $fields['job_id'] = $dbw->nextSequenceValue( 'job_job_id_seq' ); |
| | 115 | + $dbw->insert( 'job', $fields, $fname ); |
| | 116 | + } |
| | 117 | + |
| | 118 | + /** |
| | 119 | + * Run the job |
| | 120 | + * @return boolean success |
| | 121 | + */ |
| | 122 | + function run() { |
| | 123 | + $fname = 'Job::run'; |
| | 124 | + wfProfileIn( $fname ); |
| | 125 | + switch ( $this->command ) { |
| | 126 | + case 'refreshLinks': |
| | 127 | + $retval = $this->refreshLinks(); |
| | 128 | + break; |
| | 129 | + default: |
| | 130 | + $this->error = "Invalid job type {$this->command}, ignoring"; |
| | 131 | + wfDebug( $this->error . "\n" ); |
| | 132 | + $retval = false; |
| | 133 | + } |
| | 134 | + wfProfileOut( $fname ); |
| | 135 | + return $retval; |
| | 136 | + } |
| | 137 | + |
| | 138 | + /** |
| | 139 | + * Run a refreshLinks job |
| | 140 | + * @return boolean success |
| | 141 | + */ |
| | 142 | + function refreshLinks() { |
| | 143 | + global $wgParser; |
| | 144 | + |
| | 145 | + $dbw =& wfGetDB( DB_MASTER ); |
| | 146 | + |
| | 147 | + $linkCache =& LinkCache::singleton(); |
| | 148 | + $linkCache->clear(); |
| | 149 | + |
| | 150 | + if ( is_null( $this->title ) ) { |
| | 151 | + $this->error = "refreshLinks: Invalid title"; |
| | 152 | + return false; |
| | 153 | + } |
| | 154 | + |
| | 155 | + $revision = Revision::newFromTitle( $this->title ); |
| | 156 | + if ( !$revision ) { |
| | 157 | + $this->error = 'refreshLinks: Article not found "' . $this->title->getPrefixedDBkey() . '"'; |
| | 158 | + return false; |
| | 159 | + } |
| | 160 | + |
| | 161 | + $options = new ParserOptions; |
| | 162 | + $parserOutput = $wgParser->parse( $revision->getText(), $this->title, $options, true, true, $revision->getId() ); |
| | 163 | + $update = new LinksUpdate( $this->title, $parserOutput ); |
| | 164 | + $update->doUpdate(); |
| | 165 | + return true; |
| | 166 | + } |
| | 167 | + |
| | 168 | + function toString() { |
| | 169 | + if ( is_object( $this->title ) ) { |
| | 170 | + $s = "{$this->command} " . $this->title->getPrefixedDBkey(); |
| | 171 | + if ( $this->params !== '' ) { |
| | 172 | + $s .= ', ' . $this->params; |
| | 173 | + } |
| | 174 | + return $s; |
| | 175 | + } else { |
| | 176 | + return "{$this->command} {$this->params}"; |
| | 177 | + } |
| | 178 | + } |
| | 179 | + |
| | 180 | + function getLastError() { |
| | 181 | + return $this->error; |
| | 182 | + } |
| | 183 | +} |
| Property changes on: trunk/phase3/includes/JobQueue.php |
| ___________________________________________________________________ |
| Added: svn:eol-style |
| 1 | 184 | + native |
| Added: svn:keywords |
| 2 | 185 | + Author Date Id Revision |
| Index: trunk/phase3/includes/Wiki.php |
| — | — | @@ -249,6 +249,7 @@ |
| 250 | 250 | function finalCleanup ( &$deferredUpdates, &$loadBalancer, &$output ) { |
| 251 | 251 | wfProfileIn( 'MediaWiki::finalCleanup' ); |
| 252 | 252 | $this->doUpdates( $deferredUpdates ); |
| | 253 | + $this->doJobs(); |
| 253 | 254 | $loadBalancer->saveMasterPos(); |
| 254 | 255 | # Now commit any transactions, so that unreported errors after output() don't roll back the whole thing |
| 255 | 256 | $loadBalancer->commitAll(); |
| — | — | @@ -268,6 +269,38 @@ |
| 269 | 270 | } |
| 270 | 271 | wfProfileOut( 'MediaWiki::doUpdates' ); |
| 271 | 272 | } |
| | 273 | + |
| | 274 | + /** |
| | 275 | + * Do a job from the job queue |
| | 276 | + */ |
| | 277 | + function doJobs() { |
| | 278 | + global $wgJobLogFile, $wgJobRunRate; |
| | 279 | + |
| | 280 | + if ( $wgJobRunRate <= 0 ) { |
| | 281 | + return; |
| | 282 | + } |
| | 283 | + if ( $wgJobRunRate < 1 ) { |
| | 284 | + $max = mt_getrandmax(); |
| | 285 | + if ( mt_rand( 0, $max ) < $max * $wgJobRunRate ) { |
| | 286 | + return; |
| | 287 | + } |
| | 288 | + $n = 1; |
| | 289 | + } else { |
| | 290 | + $n = intval( $wgJobRunRate ); |
| | 291 | + } |
| | 292 | + |
| | 293 | + require_once( 'JobQueue.php' ); |
| | 294 | + |
| | 295 | + while ( $n-- && false != ($job = Job::pop())) { |
| | 296 | + $output = $job->toString() . "\n"; |
| | 297 | + if ( !$job->run() ) { |
| | 298 | + $output .= "Error: " . $job->getLastError() . "\n"; |
| | 299 | + } |
| | 300 | + if ( $wgJobLogFile ) { |
| | 301 | + error_log( $output, 3, $wgJobLogFile ); |
| | 302 | + } |
| | 303 | + } |
| | 304 | + } |
| 272 | 305 | |
| 273 | 306 | /** |
| 274 | 307 | * Ends this task peacefully |
| Index: trunk/phase3/includes/LinksUpdate.php |
| — | — | @@ -15,7 +15,6 @@ |
| 16 | 16 | */ |
| 17 | 17 | var $mId, # Page ID of the article linked from |
| 18 | 18 | $mTitle, # Title object of the article linked from |
| 19 | | - $mParserOutput, # Parser output containing the links to be inserted into the database |
| 20 | 19 | $mLinks, # Map of title strings to IDs for the links in the document |
| 21 | 20 | $mImages, # DB keys of the images used, in the array key only |
| 22 | 21 | $mTemplates, # Map of title strings to IDs for the template references, including broken ones |
| — | — | @@ -47,14 +46,12 @@ |
| 48 | 47 | } |
| 49 | 48 | $this->mTitle = $title; |
| 50 | 49 | $this->mId = $title->getArticleID(); |
| 51 | | - $this->mParserOutput = $parserOutput; |
| 52 | 50 | |
| 53 | | - // Shortcut aliases |
| 54 | | - $this->mLinks =& $this->mParserOutput->getLinks(); |
| 55 | | - $this->mImages =& $this->mParserOutput->getImages(); |
| 56 | | - $this->mTemplates =& $this->mParserOutput->getTemplates(); |
| 57 | | - $this->mExternals =& $this->mParserOutput->getExternalLinks(); |
| 58 | | - $this->mCategories =& $this->mParserOutput->getCategories(); |
| | 51 | + $this->mLinks = $parserOutput->getLinks(); |
| | 52 | + $this->mImages = $parserOutput->getImages(); |
| | 53 | + $this->mTemplates = $parserOutput->getTemplates(); |
| | 54 | + $this->mExternals = $parserOutput->getExternalLinks(); |
| | 55 | + $this->mCategories = $parserOutput->getCategories(); |
| 59 | 56 | |
| 60 | 57 | } |
| 61 | 58 | |
| — | — | @@ -79,11 +76,6 @@ |
| 80 | 77 | $this->incrTableUpdate( 'pagelinks', 'pl', $this->getLinkDeletions( $existing ), |
| 81 | 78 | $this->getLinkInsertions( $existing ) ); |
| 82 | 79 | |
| 83 | | - # Template links |
| 84 | | - $existing = $this->getExistingTemplates(); |
| 85 | | - $this->incrTableUpdate( 'templatelinks', 'tl', $this->getTemplateDeletions( $existing ), |
| 86 | | - $this->getTemplateInsertions( $existing ) ); |
| 87 | | - |
| 88 | 80 | # Image links |
| 89 | 81 | $existing = $this->getExistingImages(); |
| 90 | 82 | $this->incrTableUpdate( 'imagelinks', 'il', $this->getImageDeletions( $existing ), |
| — | — | @@ -93,7 +85,19 @@ |
| 94 | 86 | $existing = $this->getExistingExternals(); |
| 95 | 87 | $this->incrTableUpdate( 'externallinks', 'el', $this->getExternalDeletions( $existing ), |
| 96 | 88 | $this->getExternalInsertions( $existing ) ); |
| 97 | | - |
| | 89 | + |
| | 90 | + # Template links |
| | 91 | + $existing = $this->getExistingTemplates(); |
| | 92 | + $this->incrTableUpdate( 'templatelinks', 'tl', $this->getTemplateDeletions( $existing ), |
| | 93 | + $this->getTemplateInsertions( $existing ) ); |
| | 94 | + |
| | 95 | + # Refresh links of all pages including this page |
| | 96 | + $tlto = $this->mTitle->getTemplateLinksTo(); |
| | 97 | + if ( count( $tlto ) ) { |
| | 98 | + require_once( 'JobQueue.php' ); |
| | 99 | + Job::queueLinksJobs( $tlto ); |
| | 100 | + } |
| | 101 | + |
| 98 | 102 | # Category links |
| 99 | 103 | $existing = $this->getExistingCategories(); |
| 100 | 104 | $this->incrTableUpdate( 'categorylinks', 'cl', $this->getCategoryDeletions( $existing ), |
| — | — | @@ -117,9 +121,17 @@ |
| 118 | 122 | $fname = 'LinksUpdate::doDumbUpdate'; |
| 119 | 123 | wfProfileIn( $fname ); |
| 120 | 124 | |
| | 125 | + # Refresh category pages |
| 121 | 126 | $existing = $this->getExistingCategories(); |
| 122 | 127 | $categoryUpdates = array_diff_assoc( $existing, $this->mCategories ) + array_diff_assoc( $this->mCategories, $existing ); |
| 123 | 128 | |
| | 129 | + # Refresh links of all pages including this page |
| | 130 | + $tlto = $this->mTitle->getTemplateLinksTo(); |
| | 131 | + if ( count( $tlto ) ) { |
| | 132 | + require_once( 'JobQueue.php' ); |
| | 133 | + Job::queueLinksJobs( $tlto ); |
| | 134 | + } |
| | 135 | + |
| 124 | 136 | $this->dumbTableUpdate( 'pagelinks', $this->getLinkInsertions(), 'pl_from' ); |
| 125 | 137 | $this->dumbTableUpdate( 'imagelinks', $this->getImageInsertions(), 'il_from' ); |
| 126 | 138 | $this->dumbTableUpdate( 'categorylinks', $this->getCategoryInsertions(), 'cl_from' ); |
| Index: trunk/phase3/includes/DefaultSettings.php |
| — | — | @@ -1865,4 +1865,18 @@ |
| 1866 | 1866 | */ |
| 1867 | 1867 | $wgAllowCategorizedRecentChanges = false ; |
| 1868 | 1868 | |
| | 1869 | +/** |
| | 1870 | + * Number of jobs to perform per request. May be less than one in which case |
| | 1871 | + * jobs are performed probabalistically. If this is zero, jobs will not be done |
| | 1872 | + * during ordinary apache requests. In this case, maintenance/doJobs.php should |
| | 1873 | + * be run periodically. |
| | 1874 | + */ |
| | 1875 | +$wgJobRunRate = 1; |
| | 1876 | + |
| | 1877 | +/** |
| | 1878 | + * Log file for job execution |
| | 1879 | + */ |
| | 1880 | +$wgJobLogFile = false; |
| | 1881 | + |
| | 1882 | + |
| 1869 | 1883 | ?> |