3 namespace Rapsys\AirBundle\Command
; 
   5 use Doctrine\Bundle\DoctrineBundle\Command\DoctrineCommand
; 
   6 use Symfony\Component\Console\Input\InputInterface
; 
   7 use Symfony\Component\Console\Output\OutputInterface
; 
   8 use Symfony\Component\Filesystem\Exception\IOExceptionInterface
; 
   9 use Symfony\Component\Filesystem\Filesystem
; 
  10 use Rapsys\AirBundle\Entity\Session
; 
  12 class WeatherCommand 
extends DoctrineCommand 
{ 
  13         //Set failure constant 
  16         ///Set success constant 
  21                 //Mostly useless in fact 
  23                 //Required to simplify simplexml transition 
  25                 //Required to avoid xml errors 
  26                 'quote-nbsp' => false, 
  27                 //Required to fix code 
  31         ///Set accuweather uris 
  32         private $accuweather = [ 
  35                         75001 => 'https://www.accuweather.com/en/fr/paris-01-louvre/75001/hourly-weather-forecast/179142_pc?day=', 
  36                         75005 => 'https://www.accuweather.com/en/fr/paris-05-pantheon/75005/hourly-weather-forecast/179146_pc?day=', 
  37                         75007 => 'https://www.accuweather.com/en/fr/paris-07-palais-bourbon/75007/hourly-weather-forecast/179148_pc?day=', 
  38                         75009 => 'https://www.accuweather.com/en/fr/paris-09-opera/75009/hourly-weather-forecast/179150_pc?day=', 
  39                         75015 => 'https://www.accuweather.com/en/fr/paris-15-vaugirard/75015/hourly-weather-forecast/179156_pc?day=', 
  40                         75019 => 'https://www.accuweather.com/en/fr/paris-19-buttes-chaumont/75019/hourly-weather-forecast/179160_pc?day=', 
  41                         75116 => 'https://www.accuweather.com/en/fr/paris-16-passy/75116/hourly-weather-forecast/179246_pc?day=' 
  45                         75001 => 'https://www.accuweather.com/en/fr/paris-01-louvre/75001/daily-weather-forecast/179142_pc', 
  46                         75005 => 'https://www.accuweather.com/en/fr/paris-05-pantheon/75005/daily-weather-forecast/179146_pc', 
  47                         75007 => 'https://www.accuweather.com/en/fr/paris-07-palais-bourbon/75007/daily-weather-forecast/179148_pc', 
  48                         75009 => 'https://www.accuweather.com/en/fr/paris-09-opera/75009/daily-weather-forecast/179150_pc', 
  49                         75015 => 'https://www.accuweather.com/en/fr/paris-15-vaugirard/75015/daily-weather-forecast/179156_pc', 
  50                         75019 => 'https://www.accuweather.com/en/fr/paris-19-buttes-chaumont/75019/daily-weather-forecast/179160_pc', 
  51                         75116 => 'https://www.accuweather.com/en/fr/paris-16-passy/75116/daily-weather-forecast/179246_pc' 
  58         ///Configure attribute command 
  59         protected function configure() { 
  63                         ->setName('rapsysair:weather') 
  64                         //Set description shown with bin/console list 
  65                         ->setDescription('Updates session rain and temperature fields') 
  66                         //Set description shown with bin/console --help airlibre:attribute 
  67                         ->setHelp('This command updates session rain and temperature fields in next three days') 
  68                         //Add daily and hourly aliases 
  69                         ->setAliases(['rapsysair:weather:daily', 'rapsysair:weather:hourly']); 
  72         ///Process the attribution 
  73         protected function execute(InputInterface 
$input, OutputInterface 
$output) { 
  75                 $doctrine = $this->getDoctrine(); 
  78                 $manager = $doctrine->getManager(); 
  89                 //Process hourly accuweather 
  90                 if (($command = $input->getFirstArgument()) == 'rapsysair:weather:hourly' || $command == 'rapsysair:weather') { 
  91                         //Fetch hourly sessions to attribute 
  92                         $types['hourly'] = $doctrine->getRepository(Session
::class)->findAllPendingHourlyWeather(); 
  94                         //Iterate on each session 
  95                         foreach($types['hourly'] as $sessionId => $session) { 
  97                                 $zipcode = $session->getLocation()->getZipcode(); 
 100                                 $start = $session->getStart(); 
 103                                 $day = $start->diff((new \
DateTime('now'))->setTime(0, 0, 0))->d + 
1; 
 105                                 //Check if zipcode is set 
 106                                 if (!isset($zipcodes[$zipcode])) { 
 107                                         $zipcodes[$zipcode] = []; 
 110                                 //Check if zipcode date is set 
 111                                 if (!isset($zipcodes[$zipcode][$day])) { 
 112                                         $zipcodes[$zipcode][$day] = [ $sessionId => $sessionId ]; 
 114                                         $zipcodes[$zipcode][$day][$sessionId] = $sessionId; 
 118                                 $stop = $session->getStop(); 
 121                                 $day = $stop->diff((new \
DateTime('now'))->setTime(0, 0, 0))->d + 
1; 
 123                                 //Check if zipcode date is set 
 124                                 if (!isset($zipcodes[$zipcode][$day])) { 
 125                                         $zipcodes[$zipcode][$day] = [ $sessionId => $sessionId ]; 
 127                                         $zipcodes[$zipcode][$day][$sessionId] = $sessionId; 
 132                 //Process daily accuweather 
 133                 if ($command == 'rapsysair:weather:daily' || $command == 'rapsysair:weather') { 
 134                         //Fetch daily sessions to attribute 
 135                         $types['daily'] = $doctrine->getRepository(Session
::class)->findAllPendingDailyWeather(); 
 137                         //Iterate on each session 
 138                         foreach($types['daily'] as $sessionId => $session) { 
 140                                 $zipcode = $session->getLocation()->getZipcode(); 
 143                                 $start = $session->getStart(); 
 148                                 //Check if zipcode is set 
 149                                 if (!isset($zipcodes[$zipcode])) { 
 150                                         $zipcodes[$zipcode] = []; 
 153                                 //Check if zipcode date is set 
 154                                 if (!isset($zipcodes[$zipcode][$day])) { 
 155                                         $zipcodes[$zipcode][$day] = [ $sessionId => $sessionId ]; 
 157                                         $zipcodes[$zipcode][$day][$sessionId] = $sessionId; 
 163                 $filesystem = new Filesystem(); 
 166                 //XXX: worst case scenario we have 3 files per zipcode 
 167                 if (!is_dir($tmpdir = sys_get_temp_dir().'/accuweather')) { 
 170                             $filesystem->mkdir($tmpdir, 0775); 
 171                         } catch (IOExceptionInterface 
$exception) { 
 173                                 echo 'Create dir '.$exception->getPath().' failed'."\n"; 
 186                 //Iterate on zipcodes 
 187                 foreach($zipcodes as $zipcode => $days) { 
 189                         foreach($days as $day => $null) { 
 190                                 //Try to load content from cache 
 191                                 if (!is_file($file = $tmpdir.'/'.$zipcode.'.'.$day.'.html') || stat($file)['ctime'] <= (time() - ($day == 'daily' ? 4 : 2)*3600) || ($content = file_get_contents($file)) === false) { 
 192                                         //Prevent timing detection 
 193                                         //XXX: from 0.1 to 5 seconds 
 194                                         usleep(rand(1,50) * 100000);  
 197                                         //TODO: for daily we may load data for requested quarter of the day 
 198                                         $content = $this->curl_get($day == 'daily' ? $this->accuweather
['daily'][$zipcode] : $this->accuweather
['hourly'][$zipcode].$day); 
 201                                         if (file_put_contents($tmpdir.'/'.$zipcode.'.'.$day.'.html', $content) === false) { 
 203                                                 echo 'Write to '.$tmpdir.'/'.$zipcode.'.'.$day.'.html failed'."\n"; 
 211                                 $tidy->parseString($content, $this->config
, 'utf8'); 
 214                                 //XXX: don't care about theses errors, tidy is here to fix... 
 215                                 #if (!empty($tidy->errorBuffer)) { 
 216                                 #       var_dump($tidy->errorBuffer); 
 217                                 #       die('Tidy errors'); 
 221                                 //XXX: trash all xmlns= broken tags 
 222                                 $sx = new \
SimpleXMLElement(str_replace(['xmlns=', 'xlink:href='], ['xns=', 'href='], $tidy)); 
 225                                 if ($day == 'daily') { 
 226                                         //Iterate on each link containing data 
 227                                         foreach($sx->xpath('//a[@class="daily-forecast-card"]') as $node) { 
 229                                                 $dsm = trim($node->div
[0]->h2
[0]->span
[1]); 
 232                                                 $temperature = str_replace('°', '', $node->div
[0]->div
[0]->span
[0]); 
 235                                                 $rainrisk = str_replace('%', '', trim($node->div
[2]))/100; 
 238                                                 $data[$zipcode][$dsm]['daily'] = [ 
 239                                                         'temperature' => $temperature, 
 240                                                         'rainrisk' => $rainrisk 
 245                                         //Iterate on each div containing data 
 246                                         #(string)$sx->xpath('//div[@class="hourly-card-nfl"]')[0]->attributes()->value 
 247                                         #/html/body/div[1]/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div/h2/span[1] 
 248                                         foreach($sx->xpath('//div[@data-shared="false"]') as $node) { 
 250                                                 $hour = trim($node->div
[0]->div
[0]->h2
[0]->span
[0]); 
 253                                                 $dsm = trim($node->div
[0]->div
[0]->h2
[0]->span
[1]); 
 256                                                 $temperature = str_replace('°', '', $node->div
[0]->div
[0]->div
[0]); 
 259                                                 $realfeel = str_replace(['RealFeel® ', '°'], '', trim($node->div
[0]->div
[0]->span
[0])); 
 262                                                 $rainrisk = str_replace('%', '', trim($node->div
[0]->div
[0]->div
[1]))/100; 
 264                                                 //Label is Rain when we have a rainfall 
 265                                                 if (($pluviolabel = trim($node->div
[1]->div
[0]->div
[0]->div
[1]->p
[1])) == 'Rain') { 
 267                                                         $rainfall = str_replace(' mm', '', $node->div
[1]->div
[0]->div
[0]->div
[1]->p
[1]->span
[0]); 
 268                                                 //Cloud Cover, no rainfall 
 270                                                         //Set rainfall to 0 (mm) 
 275                                                 $data[$zipcode][$dsm][$hour] = [ 
 276                                                         'temperature' => $temperature, 
 277                                                         'realfeel' => $realfeel, 
 278                                                         'rainrisk' => $rainrisk, 
 279                                                         'rainfall' => $rainfall 
 290                 foreach($types as $type => $sessions) { 
 291                         //Iterate on each type 
 292                         foreach($sessions as $sessionId => $session) { 
 294                                 $zipcode = $session->getLocation()->getZipcode(); 
 297                                 $start = $session->getStart(); 
 300                                 if ($type == 'daily') { 
 302                                         $period = [ $start ]; 
 306                                         $stop = $session->getStop(); 
 309                                         $period = new \
DatePeriod( 
 312                                                 //Iterate on each hour 
 313                                                 new \
DateInterval('PT1H'), 
 314                                                 //End with begin + length 
 324                                         'realfeelmin' => null, 
 325                                         'realfeelmax' => null, 
 327                                         'temperaturemin' => null, 
 328                                         'temperaturemax' => null 
 331                                 //Iterate on the period 
 332                                 foreach($period as $time) { 
 334                                         $dsm = $time->format('d/m'); 
 337                                         $hour = $type=='daily'?$type:$time->format('H'); 
 339                                         //Check data availability 
 340                                         //XXX: should never happen 
 341                                         #if (!isset($data[$zipcode][$dsm][$hour])) { 
 342                                         #       //Skip unavailable data 
 347                                         $info = $data[$zipcode][$dsm][$hour]; 
 349                                         //Check if rainrisk is higher 
 350                                         if ($meteo['rainrisk'] === null || $info['rainrisk'] > $meteo['rainrisk']) { 
 351                                                 //Set highest rain risk 
 352                                                 $meteo['rainrisk'] = floatval($info['rainrisk']); 
 355                                         //Check if rainfall is set 
 356                                         if (isset($info['rainfall'])) { 
 358                                                 $meteo['rainfall'] +
= floatval($info['rainfall']); 
 362                                         $meteo['temperature'][$hour] = $info['temperature']; 
 365                                         if ($type != 'daily') { 
 366                                                 //Check min temperature 
 367                                                 if ($meteo['temperaturemin'] === null || $info['temperature'] < $meteo['temperaturemin']) { 
 368                                                         $meteo['temperaturemin'] = floatval($info['temperature']); 
 371                                                 //Check max temperature 
 372                                                 if ($meteo['temperaturemax'] === null || $info['temperature'] > $meteo['temperaturemax']) { 
 373                                                         $meteo['temperaturemax'] = floatval($info['temperature']); 
 377                                         //Check if realfeel is set 
 378                                         if (isset($info['realfeel'])) { 
 380                                                 $meteo['realfeel'][$hour] = $info['realfeel']; 
 383                                                 if ($meteo['realfeelmin'] === null || $info['realfeel'] < $meteo['realfeelmin']) { 
 384                                                         $meteo['realfeelmin'] = floatval($info['realfeel']); 
 388                                                 if ($meteo['realfeelmax'] === null || $info['realfeel'] > $meteo['realfeelmax']) { 
 389                                                         $meteo['realfeelmax'] = floatval($info['realfeel']); 
 394                                 //Check if rainfall is set and differ 
 395                                 if ($session->getRainfall() !== $meteo['rainfall']) { 
 397                                         $session->setRainfall($meteo['rainfall']); 
 400                                 //Check if rainrisk differ 
 401                                 if ($session->getRainrisk() !== $meteo['rainrisk']) { 
 403                                         $session->setRainrisk($meteo['rainrisk']); 
 406                                 //Check realfeel array 
 407                                 if ($meteo['realfeel'] !== []) { 
 409                                         $realfeel = floatval(round(array_sum($meteo['realfeel'])/count($meteo['realfeel']),1)); 
 411                                         //Check if realfeel differ 
 412                                         if ($session->getRealfeel() !== $realfeel) { 
 413                                                 //Set average realfeel 
 414                                                 #$meteo['realfeel'] = array_sum($meteo['realfeel'])/count($meteo['realfeel']); 
 415                                                 $session->setRealfeel($realfeel); 
 418                                         //Check if realfeelmin differ 
 419                                         if ($session->getRealfeelmin() !== $meteo['realfeelmin']) { 
 421                                                 $session->setRealfeelmin($meteo['realfeelmin']); 
 424                                         //Check if realfeelmax differ 
 425                                         if ($session->getRealfeelmax() !== $meteo['realfeelmax']) { 
 427                                                 $session->setRealfeelmax($meteo['realfeelmax']); 
 431                                 //Check temperature array 
 432                                 if ($meteo['temperature'] !== []) { 
 433                                         //Compute temperature 
 434                                         $temperature = floatval(round(array_sum($meteo['temperature'])/count($meteo['temperature']),1)); 
 436                                         //Check if temperature differ 
 437                                         if ($session->getTemperature() !== $temperature) { 
 438                                                 //Set average temperature 
 439                                                 #$meteo['temperature'] = array_sum($meteo['temperature'])/count($meteo['temperature']); 
 440                                                 $session->setTemperature($temperature); 
 443                                         //Check if temperaturemin differ 
 444                                         if ($session->getTemperaturemin() !== $meteo['temperaturemin']) { 
 446                                                 $session->setTemperaturemin($meteo['temperaturemin']); 
 449                                         //Check if temperaturemax differ 
 450                                         if ($session->getTemperaturemax() !== $meteo['temperaturemax']) { 
 452                                                 $session->setTemperaturemax($meteo['temperaturemax']); 
 458                 //Flush to get the ids 
 465                 return self
::SUCCESS
; 
 471          * @return bool|void Return success or exit 
 473         function curl_init() { 
 475                 if (($this->ch 
= curl_init()) === false) { 
 477                         echo 'Curl init failed: '.curl_error($this->ch
)."\n"; 
 488                                         CURLOPT_HTTP_VERSION 
=> CURL_HTTP_VERSION_2_0
, 
 490                                         CURLOPT_HTTPHEADER 
=> [ 
 491                                                 //XXX: it seems that you can disable akamai fucking protection with Pragma: akamai-x-cache-off 
 492                                                 //XXX: see https://support.globaldots.com/hc/en-us/articles/115003996705-Akamai-Pragma-Headers-overview 
 493                                                 #'Pragma: akamai-x-cache-off', 
 494                                                 //XXX: working curl command 
 495                                                 #curl --http2 --cookie file.jar --cookie-jar file.jar -v -i -k -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' -H 'Accept-Language: en-GB,en;q=0.9' -H 'Cache-Control: no-cache' -H 'Connection: keep-alive' -H 'Host: www.accuweather.com' -H 'Pragma: no-cache' -H 'Upgrade-Insecure-Requests: 1' -H 'User-Agent: Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36' 'https://www.accuweather.com/' 
 497                                                 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', 
 498                                                 //Set accept language 
 499                                                 'Accept-Language: en-GB,en;q=0.9', 
 501                                                 'Cache-Control: no-cache', 
 502                                                 //Keep connection alive 
 503                                                 'Connection: keep-alive', 
 506                                                 //Force secure requests 
 507                                                 'Upgrade-Insecure-Requests: 1', 
 509                                                 'User-Agent: Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36', 
 510                                                 //Force akamai cookie 
 511                                                 //XXX: seems to come from http request 
 515                                         CURLOPT_COOKIEFILE 
=> '', 
 516                                         //Disable location following 
 517                                         CURLOPT_FOLLOWLOCATION 
=> false, 
 519                                         #CURLOPT_URL => $url = 'https://www.accuweather.com/', 
 521                                         CURLOPT_HEADER 
=> true, 
 523                                         CURLOPT_RETURNTRANSFER 
=> true, 
 526                                         CURLINFO_HEADER_OUT 
=> true 
 531                         echo 'Curl setopt array failed: '.curl_error($this->ch
)."\n"; 
 543          * @return string|void Return url content or exit 
 545         function curl_get($url) { 
 547                 if (curl_setopt($this->ch
, CURLOPT_URL
, $url) === false) { 
 549                         echo 'Setopt for '.$url.' failed: '.curl_error($this->ch
)."\n"; 
 552                         curl_close($this->ch
); 
 558                 //Check return status 
 559                 if (($response = curl_exec($this->ch
)) === false) { 
 561                         echo 'Get for '.$url.' failed: '.curl_error($this->ch
)."\n"; 
 563                         //Display sent headers 
 564                         var_dump(curl_getinfo($this->ch
, CURLINFO_HEADER_OUT
)); 
 570                         curl_close($this->ch
); 
 577                 if (empty($hs = curl_getinfo($this->ch
, CURLINFO_HEADER_SIZE
))) { 
 579                         echo 'Getinfo for '.$url.' failed: '.curl_error($this->ch
)."\n"; 
 581                         //Display sent headers 
 582                         var_dump(curl_getinfo($this->ch
, CURLINFO_HEADER_OUT
)); 
 588                         curl_close($this->ch
); 
 595                 if (empty($header = substr($response, 0, $hs))) { 
 597                         echo 'Header for '.$url.' empty: '.curl_error($this->ch
)."\n"; 
 599                         //Display sent headers 
 600                         var_dump(curl_getinfo($this->ch
, CURLINFO_HEADER_OUT
)); 
 606                         curl_close($this->ch
); 
 612                 //Check request success 
 613                 if (strlen($header) <= 10 || substr($header, 0, 10) !== 'HTTP/2 200') { 
 615                         echo 'Status for '.$url.' failed: '.curl_error($this->ch
)."\n"; 
 617                         //Display sent headers 
 618                         var_dump(curl_getinfo($this->ch
, CURLINFO_HEADER_OUT
)); 
 624                         curl_close($this->ch
); 
 631                 return substr($response, $hs); 
 637          * @return bool Return success or failure 
 639         function curl_close() { 
 640                 return curl_close($this->ch
);