pxo=new parseXOVER; $this->xoverinterval=$GLOBALS['CONF_filler']['xoverinterval']; $this->logrejects=$GLOBALS['CONF_filler']['logrejects']; } function doXOVER($gi,$hdrlimit,$startHdr,$allhdr){ if($this->sql->toggleupdatelock(true,$gi['groupid'])){ $this->hdradded=0; $this->lasthdrID=0; $this->indexID=0; if($this->live){echo 'Reading Headers in '.$gi['group'].':
';}flush(); //check socket status, try to open again if it is failed.... if(!$this->recoverNNTP(0)){ if($this->live){echo "Could not recover NNTP connection!";flush();} $this->errorLog['l'][]=0; $this->errorLog['e'][]='Could not recover NNTP connection! parseXOVER::doXOVER: '.$gi['group']; $this->sql->toggleupdatelock(false,$gi['groupid']); return false; } //change group, decide where to start and end $xend=$this->initGroup($gi,$hdrlimit,$startHdr,$allhdr); //couldnt change group! if(!$xend){ if($this->live){echo "This group not on the server!";flush();} $this->errorLog['l'][]=0; $this->errorLog['e'][]='Group may not be on server: '.$gi['group']; $this->sql->toggleupdatelock(false,$gi['groupid']); return false; } //get the header and parse into the database $this->retGroup($gi['groupid'],$xend); if($this->live){echo "
Done!
";flush();} $this->errorLog['l'][]=2; $this->errorLog['e'][]=$gi['group'].": got ".$this->hdradded." headers of (up to) ".($this->ns->group['lastID'] - $this->ns->group['firstID']).', '.($this->ns->group['lastID'] - $this->pxo->xover['id']).' unprocessed.'; //only update group stats if there are new headers added if($this->hdradded>0){$this->sql->UPDATEGROUPSTATS($gi['groupid']);} //togglt lock now sets the update time stamp if we didnt do that in updategroupstats $this->sql->toggleupdatelock(false,$gi['groupid']); return true; }else{ $this->errorLog['l'][]=1; $this->errorLog['e'][]='Header update already locked for '.$gi['group'].'. This means another process is importing headers for this group at the same time, or a previous process crashed before finishing. You can manually unlock the group from the administration page.'; return false; } } function retGroup($groupid,$XEND){ $this->hdradded=0; $this->pxo->groupid=$groupid; //test to see that there are any header that need to be fetched... //if the last ID scanned is the same as the last id in the group, then skip this group if($this->lasthdrID==$this->ns->group['lastID']){return;} //each time try for xxxx headers //loop untill at least the requested number are found //repeated to the total requested number of records (or end of data) while($this->hdradded<$XEND){//need to add a way out of this loop.... //do the group xover retrieval if(!$this->retloop()){break;}//break out on failed connection, prevents us from trying over and over on dead connecton.. //if current header is >= the last header in the group we need to break if($this->lasthdrID>=$this->ns->group['lastID']){ if($this->live){echo "Reached last header!";flush();} break; } //if indexID is greater than lastgroupID, but lastheaderID is NOT, //some groups have no messages, but report first/last anyways //so we store the lastID as the lastheaderID where we will begin next time if($this->indexID>=$this->ns->group['lastID']){ if($this->live){echo "Probably no headers!";flush();} $this->lasthdrID=$this->ns->group['lastID']; break; } } //flush the matching caches once per group. $this->pxo->clearfilescache(); //save the last header ID, we start here +1 next time. if(!$this->sql->savelastheaderid($groupid, $this->lasthdrID)){ $this->errorLog['l'][]=1; $this->errorLog['e'][]='New header ('.$this->lasthdrID.','.$this->group.') ID less than last ID!'; } }//function function retloop(){ //start the xover retrieve if(!$this->ns->xover($this->indexID+1,$this->indexID+$this->xoverinterval)){ $this->nntp_error('filler::retloop::writeXOVER: '); return false; } if($this->live){$time_start = $this->microtime_float();} //process the xover data while($this->proc_xover()){}; //push the cache to the database $this->pxo->flushcache($this->sql); //increment the index, we start here next time. $this->indexID+=$this->xoverinterval; //live feedback for admin interface if($this->live){ $time_end = $this->microtime_float(); $time = $time_end - $time_start; echo $this->hdradded.' in '.round($time,2).'s('.$this->indexID.'/'.$this->ns->group['lastID'].')|';flush(); } //check the connection status here //try to open new connection and retry if fail... if ($this->ns->nntp_con != false){ return true; }else{ if(!$this->recoverNNTP(1,$this->group)){ $this->nntp_error('recover in retloop()'); return false; } return true; } } function proc_xover(){ //read next line of xover $xo=$this->ns->readNNTPline(); //line returns false (nntp error!) if(!$xo){ $this->nntp_error('proc_xover'); return false; } //line is '.' (end of xover) if(strcmp($xo,".") == 0){return false;} //we have data //parse it to array $this->pxo->XOVER2ARRAY($xo); //store the header id so we can recover from errors where we left off.... //only do this if we find a header in the group $this->lasthdrID=$this->pxo->xover['id']; //look for file segment, add it if true. if($this->pxo->findfile()){ $this->pxo->findcollection(); $this->pxo->savetoDB($this->sql); $this->hdradded++; }elseif($this->logrejects){ $this->pxo->logreject($this->sql); } return true; } function nntp_error($who){ $this->errorLog['l'][]=0; $this->errorLog['e'][]=$who.'->NNTP::'.$this->ns->errorLog; } function microtime_float(){ list($usec, $sec) = explode(" ", microtime()); return ((float)$usec + (float)$sec); } function recoverNNTP($level=0,$group=''){ //check socket status, try to open again if it is failed.... if ($this->ns->nntp_con == false){ if(!$this->ns->startNNTPconnection()){ //couldnt open socket $this->nntp_error('Failed to (re)open the NNTP connection in filler::recoverNNTP'); return false; } } //recover the group if($level>0){ $this->ns->changegroup($group); if(!$this->ns->group['status']){ $this->nntp_error('recoverNNTP-change groups'); return false; } } return true; } function initGroup($gi,$hdrlimit,$startHdr,$allhdr){ if(!$this->ns->changegroup($gi['group'])){ $this->nntp_error('initgroup-GROUP'); return false; } if(!$this->ns->group['status']){ $this->errorLog['l'][]=1; $this->errorLog['e'][]='Error changing groups: '.$this->ns->errorLog; return false; } //if firstID is greater, start there, else use ID+1 if($gi['lastheaderid']<$this->ns->group['firstID']){ $lastheaderread=$this->ns->group['lastID']-$GLOBALS['CONF_cron']['maxnewgroupheaders']; //a check to make sure we dont end up where there are no messages..... //this prevents jordy from doing stupid things :) if($lastheaderread<$this->ns->group['firstID']){ $lastheaderread=$this->ns->group['firstID']; } }else{ $lastheaderread=$gi['lastheaderid']; } //if the specified start is further forward than the last ID, then start there. if($lastheaderread<$startHdr){$lastheaderread=$startHdr;} //if we want all the headers, set the limit crazy high, in this case the id limit of the last message ID //the while loop will break because we exceed the last ID if($allhdr){ $hdrlimit=$this->ns->group['lastID']; //progress message for admin if($this->live){echo 'Reading all Headers!!! This may take a while!
';} } //configure the class with the correct lasthdrID... $this->lasthdrID=$lastheaderread; $this->indexID=$lastheaderread; $this->group=$gi['group']; return $hdrlimit; } /* function processNFOcache(){ //clear out the NFO cache $this->getNFO(); }*/ function processNFOcache(){ $i=0; //load list of nfo to get $nfocache=$this->sql->fetchNFOcache(); if($nfocache){ foreach($nfocache as $key=>$nc){ for($j=0;$j<$this->bodyretries;$j++){ //check socket status, try to open again if it is failed.... if(!$this->recoverNNTP(0)){ if($this->live){echo "Could not recover NNTP connection!";flush();} $this->errorLog['l'][]=0; $this->errorLog['e'][]= 'Could not recover NNTP connection! parseXOVER:processNFOcache: '.$messageid; continue; } if($this->parseBody($nc,$key)){$i++;break;}//got it, break out } } $this->errorLog['l'][]=2; $this->errorLog['e'][]=$i.' NFOs added.'; } } function parseBody($messageid,$collectionid){ //get message if(!$this->ns->body($messageid)){ $this->nntp_error('filler::parseBody:BODY(cid='.$collectionid.','.$messageid.')'); //there was an error if (substr($this->ns->errorLog,0,3) === '430'){//message not found //couldnt get the body, but delete from nfo cache.... $this->sql->updateNFO($collectionid,$url,false); return true; } return false; } //grab the body $b=$this->ns->readNNTPbulk(); if(!$b){ //there was an error $this->nntp_error('filler::parseBody:readNNTPbulk('.$messageid.')'); return false; } //pass to yenc, decode $nfotext=$this->decode($b); //save as collection ID name.txt $destination=$GLOBALS['CONF_nfo']['folder']; $filename=md5(trim($collectionid)).'.txt'; if($filename==""){ $this->errorLog['l'][]=0; $this->errorLog['e'][]="$filename there is no name present for the file to be saved!"; return false; } // Make sure the destination directory exists. if (!is_dir($destination)){ $this->errorLog['l'][]=0; $this->errorLog['e'][]= "Destination directory ($destination) does not exist."; return false; } // Write the file. if ($fp = @fopen("$destination/$filename", "wb")){ fwrite($fp, $nfotext); fclose($fp); }else{ $this->errorLog['l'][]=0; $this->errorLog['e'][]="Could not open $destination - $filename for write access."; return false; } //find preference URLs.... preg_match_all("|http://\S*[\s]|Ui",$nfotext,$urls,PREG_PATTERN_ORDER); foreach($urls[0] as $val){ $url=$val; foreach($GLOBALS['CONF_nfo']['prefurls'] as $prefurl){ if(stristr($url,$prefurl)){//note the === //found one of our preference urls, break the loop break 2; } } } //set collection bit (nfo available) to true, add web address. //remove entry from NFOcache table... $this->sql->updateNFO($collectionid,$url,true); return true; } /* * yEnc.php - yEnc PHP Class. * Copyright (c) 2003 Ryan Grove . All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the * Free Software Foundation; either version 2 of the License, or any later * version. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General * Public License for more details. * * You should have received a copy of the GNU General Public License along * with this program; if not, write to the Free Software Foundation, Inc., * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ /** * yEnc PHP Class. * * This class provides functions to encode and decode yEnc files * and strings. It meets the specifications of version 1.3 of the * yEnc working draft (http://www.yenc.org/yenc-draft.1.3.txt) * and also incorporates several unofficial (but recommended) features such * as escaping of tab, space and period (.) characters. * * @author Ryan Grove (ryan\@wonko.com) * @date March 31, 2003 * @version 1.1.0 */ function decode($string){ $encoded = array(); $header = array(); $trailer = array(); $decoded = ''; $c = ''; // Extract the yEnc string itself. preg_match("/^(=ybegin.*=yend[^$]*)$/ims", $string, $encoded); $encoded = $encoded[1]; /* // Extract the file size from the header. preg_match("/^=ybegin.*size=([^ $]+)/im", $encoded, $header); $headersize = $header[1]; // Extract the file name from the header. //we make our own file name.... preg_match("/^=ybegin.*name=([^\\r\\n]+)/im", $encoded, $header); $filename = trim($header[1]); // Extract the file size from the trailer. preg_match("/^=yend.*size=([^ $\\r\\n]+)/im", $encoded, $trailer); $trailersize = $trailer[1]; */ /* // Extract the CRC32 checksum from the trailer (if any). preg_match("/^=yend.*crc32=([^ $\\r\\n]+)/im", $encoded, $trailer); $crc = @trim(@$trailer[1]); */ // Remove the header and trailer from the string before parsing it. $encoded = preg_replace("/(^=ybegin.*\\r\\n)/im", "", $encoded, 1); //this had to be added, probably could be added to the preg_replace above $encoded = preg_replace("/(^=ypart.*\\r\\n)/im","",$encoded,1); $encoded = preg_replace("/(^=yend.*)/im", "", $encoded, 1); // Remove linebreaks from the string. $encoded = trim(str_replace("\r\n", "", $encoded)); // Make sure the header and trailer filesizes match up. //commented out because it doesn't work - there are never really problems anyways... /* if ($headersize != $trailersize){ $this->errorLog['l'][]=2; $this->errorLog['e'][]="Header ($headersize) and trailer ($trailersize) file sizes do not match. This is a violation of the yEnc specification."; }*/ // Decode $strLength = strlen($encoded); for( $i = 0; $i < $strLength; $i++){ $c = $encoded{$i}; if ($c == '='){ $i++; $decoded .= chr((ord($encoded{$i}) - 64) - 42); }else{ $decoded .= chr(ord($c) - 42); } } /* // Make sure the decoded filesize is the same as the size specified in the header. if (strlen($decoded) != $headersize){ $this->errorLog['l'][]=2; $this->errorLog['e'][]= "Header file size ($headersize) and actual file size (".strlen($decoded).") do not match. The file is probably corrupt."; } */ // Check the CRC value /* if ($crc != "" && strtolower($crc) != strtolower(sprintf("%04X", crc32($decoded)))){ $this->errorLog['l'][]=2; $this->errorLog['e'][]="CRC32 checksums do not match. The file is probably corrupt."; } */ return $decoded; } }//class /* OLD RETFUNCTION function retGroup($groupid,$group,$XBEGIN,$XEND){ //how many records to try to get each time, //repeated to the total requested number of records (or end of data) $xointerval=1000; $hdradded=0; while($hdradded<$XEND){ //each time try for xxxx headers //loop untill at least the requested number are found //this lets us give some feedback for each group //start the xover retrieve $this->ns->xover($XBEGIN,($XBEGIN+$xointerval)); $xo['line']=true; $xo['data']=''; //************************************************* //uncomment here and below to time the xover class //************************************************** //$time_start = $this->microtime_float(); while($xo['line']){ //read next line of xover $xo=$this->ns->readxover(); //is there more data? if($xo['line']){ //parse it to array $this->pxo->XOVER2ARRAY($groupid,$group,$xo['data']); //look for file segment, add it if true. if($this->pxo->findfile()){ $this->pxo->findcollection(); $this->pxo->savetoDB($this->sql); $hdradded++; }elseif($this->logrejects){ $this->pxo->logreject($this->sql); }//found file }//got xover data }//while we have xover data to parse //push the header cache to the database $this->pxo->flushcache($this->sql); //start next at end of this batch $XBEGIN+=($xointerval+1); //usefull for timing xover without ZEND //$time_end = $this->microtime_float(); //$time = $time_end - $time_start; //if($this->live){echo "$hdradded in $time|";flush();} if($this->live){echo "$hdradded|";flush();} //if start is > than header limit we need to break if($XBEGIN>$this->ns->group['lastID']){$hdradded+=1;$XEND=0;} } //flush the matching caches once per group. $this->pxo->clearfilescache(); //get the last header ID to save //we will start here +1 next time! if(!$this->sql->savelastheaderid($groupid,$this->pxo->xover['id'])){ $this->errorLog['l'][]=1; $this->errorLog['e'][]='New header ('.$this->pxo->xover['id'].','.$group.') ID less than last ID!'; } return $hdradded; }//function */ ?>