3 namespace Rapsys\AirBundle\Command
; 
   5 use Doctrine\Bundle\DoctrineBundle\Command\DoctrineCommand
; 
   6 use Symfony\Component\Console\Input\InputInterface
; 
   7 use Symfony\Component\Console\Output\OutputInterface
; 
   8 use Symfony\Component\Filesystem\Exception\IOExceptionInterface
; 
   9 use Symfony\Component\Filesystem\Filesystem
; 
  10 use Rapsys\AirBundle\Entity\Session
; 
  12 class WeatherCommand 
extends DoctrineCommand 
{ 
  13         //Set failure constant 
  16         ///Set success constant 
  21                 //Mostly useless in fact 
  23                 //Required to simplify simplexml transition 
  25                 //Required to avoid xml errors 
  26                 'quote-nbsp' => false, 
  27                 //Required to fix code 
  31         ///Set accuweather uris 
  32         private $accuweather = [ 
  35                         75001 => 'https://www.accuweather.com/en/fr/paris-01-louvre/75001/hourly-weather-forecast/179142_pc?day=', 
  36                         75004 => 'https://www.accuweather.com/en/fr/paris-04-hotel-de-ville/75004/hourly-weather-forecast/179145_pc?day=', 
  37                         75005 => 'https://www.accuweather.com/en/fr/paris-05-pantheon/75005/hourly-weather-forecast/179146_pc?day=', 
  38                         75007 => 'https://www.accuweather.com/en/fr/paris-07-palais-bourbon/75007/hourly-weather-forecast/179148_pc?day=', 
  39                         75009 => 'https://www.accuweather.com/en/fr/paris-09-opera/75009/hourly-weather-forecast/179150_pc?day=', 
  40                         75015 => 'https://www.accuweather.com/en/fr/paris-15-vaugirard/75015/hourly-weather-forecast/179156_pc?day=', 
  41                         75019 => 'https://www.accuweather.com/en/fr/paris-19-buttes-chaumont/75019/hourly-weather-forecast/179160_pc?day=', 
  42                         75116 => 'https://www.accuweather.com/en/fr/paris-16-passy/75116/hourly-weather-forecast/179246_pc?day=' 
  46                         75001 => 'https://www.accuweather.com/en/fr/paris-01-louvre/75001/daily-weather-forecast/179142_pc', 
  47                         75004 => 'https://www.accuweather.com/en/fr/paris-04-hotel-de-ville/75004/daily-weather-forecast/179145_pc', 
  48                         75005 => 'https://www.accuweather.com/en/fr/paris-05-pantheon/75005/daily-weather-forecast/179146_pc', 
  49                         75007 => 'https://www.accuweather.com/en/fr/paris-07-palais-bourbon/75007/daily-weather-forecast/179148_pc', 
  50                         75009 => 'https://www.accuweather.com/en/fr/paris-09-opera/75009/daily-weather-forecast/179150_pc', 
  51                         75015 => 'https://www.accuweather.com/en/fr/paris-15-vaugirard/75015/daily-weather-forecast/179156_pc', 
  52                         75019 => 'https://www.accuweather.com/en/fr/paris-19-buttes-chaumont/75019/daily-weather-forecast/179160_pc', 
  53                         75116 => 'https://www.accuweather.com/en/fr/paris-16-passy/75116/daily-weather-forecast/179246_pc' 
  60         ///Configure attribute command 
  61         protected function configure() { 
  65                         ->setName('rapsysair:weather') 
  66                         //Set description shown with bin/console list 
  67                         ->setDescription('Updates session rain and temperature fields') 
  68                         //Set description shown with bin/console --help airlibre:attribute 
  69                         ->setHelp('This command updates session rain and temperature fields in next three days') 
  70                         //Add daily and hourly aliases 
  71                         ->setAliases(['rapsysair:weather:daily', 'rapsysair:weather:hourly']); 
  74         ///Process the attribution 
  75         protected function execute(InputInterface 
$input, OutputInterface 
$output) { 
  77                 $doctrine = $this->getDoctrine(); 
  80                 $manager = $doctrine->getManager(); 
  91                 //Process hourly accuweather 
  92                 if (($command = $input->getFirstArgument()) == 'rapsysair:weather:hourly' || $command == 'rapsysair:weather') { 
  93                         //Fetch hourly sessions to attribute 
  94                         $types['hourly'] = $doctrine->getRepository(Session
::class)->findAllPendingHourlyWeather(); 
  96                         //Iterate on each session 
  97                         foreach($types['hourly'] as $sessionId => $session) { 
  99                                 $zipcode = $session->getLocation()->getZipcode(); 
 102                                 $start = $session->getStart(); 
 105                                 $day = $start->diff((new \
DateTime('now'))->setTime(0, 0, 0))->d + 
1; 
 107                                 //Check if zipcode is set 
 108                                 if (!isset($zipcodes[$zipcode])) { 
 109                                         $zipcodes[$zipcode] = []; 
 112                                 //Check if zipcode date is set 
 113                                 if (!isset($zipcodes[$zipcode][$day])) { 
 114                                         $zipcodes[$zipcode][$day] = [ $sessionId => $sessionId ]; 
 116                                         $zipcodes[$zipcode][$day][$sessionId] = $sessionId; 
 120                                 $stop = $session->getStop(); 
 123                                 $day = $stop->diff((new \
DateTime('now'))->setTime(0, 0, 0))->d + 
1; 
 125                                 //Check if zipcode date is set 
 126                                 if (!isset($zipcodes[$zipcode][$day])) { 
 127                                         $zipcodes[$zipcode][$day] = [ $sessionId => $sessionId ]; 
 129                                         $zipcodes[$zipcode][$day][$sessionId] = $sessionId; 
 134                 //Process daily accuweather 
 135                 if ($command == 'rapsysair:weather:daily' || $command == 'rapsysair:weather') { 
 136                         //Fetch daily sessions to attribute 
 137                         $types['daily'] = $doctrine->getRepository(Session
::class)->findAllPendingDailyWeather(); 
 139                         //Iterate on each session 
 140                         foreach($types['daily'] as $sessionId => $session) { 
 142                                 $zipcode = $session->getLocation()->getZipcode(); 
 145                                 $start = $session->getStart(); 
 150                                 //Check if zipcode is set 
 151                                 if (!isset($zipcodes[$zipcode])) { 
 152                                         $zipcodes[$zipcode] = []; 
 155                                 //Check if zipcode date is set 
 156                                 if (!isset($zipcodes[$zipcode][$day])) { 
 157                                         $zipcodes[$zipcode][$day] = [ $sessionId => $sessionId ]; 
 159                                         $zipcodes[$zipcode][$day][$sessionId] = $sessionId; 
 165                 $filesystem = new Filesystem(); 
 168                 //XXX: worst case scenario we have 3 files per zipcode 
 169                 if (!is_dir($tmpdir = sys_get_temp_dir().'/accuweather')) { 
 172                             $filesystem->mkdir($tmpdir, 0775); 
 173                         } catch (IOExceptionInterface 
$exception) { 
 175                                 echo 'Create dir '.$exception->getPath().' failed'."\n"; 
 188                 //Iterate on zipcodes 
 189                 foreach($zipcodes as $zipcode => $days) { 
 191                         foreach($days as $day => $null) { 
 192                                 //Try to load content from cache 
 193                                 if (!is_file($file = $tmpdir.'/'.$zipcode.'.'.$day.'.html') || stat($file)['ctime'] <= (time() - ($day == 'daily' ? 4 : 2)*3600) || ($content = file_get_contents($file)) === false) { 
 194                                         //Prevent timing detection 
 195                                         //XXX: from 0.1 to 5 seconds 
 196                                         usleep(rand(1,50) * 100000);  
 199                                         //TODO: for daily we may load data for requested quarter of the day 
 200                                         $content = $this->curl_get($day == 'daily' ? $this->accuweather
['daily'][$zipcode] : $this->accuweather
['hourly'][$zipcode].$day); 
 203                                         if (file_put_contents($tmpdir.'/'.$zipcode.'.'.$day.'.html', $content) === false) { 
 205                                                 echo 'Write to '.$tmpdir.'/'.$zipcode.'.'.$day.'.html failed'."\n"; 
 213                                 $tidy->parseString($content, $this->config
, 'utf8'); 
 216                                 //XXX: don't care about theses errors, tidy is here to fix... 
 217                                 #if (!empty($tidy->errorBuffer)) { 
 218                                 #       var_dump($tidy->errorBuffer); 
 219                                 #       die('Tidy errors'); 
 223                                 //XXX: trash all xmlns= broken tags 
 224                                 $sx = new \
SimpleXMLElement(str_replace(['xmlns=', 'xlink:href='], ['xns=', 'href='], $tidy)); 
 227                                 if ($day == 'daily') { 
 228                                         //Iterate on each link containing data 
 229                                         foreach($sx->xpath('//a[@class="daily-forecast-card"]') as $node) { 
 231                                                 $dsm = trim($node->div
[0]->h2
[0]->span
[1]); 
 234                                                 $temperature = str_replace('°', '', $node->div
[0]->div
[0]->span
[0]); 
 237                                                 $rainrisk = str_replace('%', '', trim($node->div
[2]))/100; 
 240                                                 $data[$zipcode][$dsm]['daily'] = [ 
 241                                                         'temperature' => $temperature, 
 242                                                         'rainrisk' => $rainrisk 
 247                                         //Iterate on each div containing data 
 248                                         #(string)$sx->xpath('//div[@class="hourly-card-nfl"]')[0]->attributes()->value 
 249                                         #/html/body/div[1]/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div/h2/span[1] 
 250                                         foreach($sx->xpath('//div[@data-shared="false"]') as $node) { 
 252                                                 $hour = trim($node->div
[0]->div
[0]->h2
[0]->span
[0]); 
 255                                                 $dsm = trim($node->div
[0]->div
[0]->h2
[0]->span
[1]); 
 258                                                 $temperature = str_replace('°', '', $node->div
[0]->div
[0]->div
[0]); 
 261                                                 $realfeel = str_replace(['RealFeel® ', '°'], '', trim($node->div
[0]->div
[0]->span
[0])); 
 264                                                 $rainrisk = str_replace('%', '', trim($node->div
[0]->div
[0]->div
[1]))/100; 
 266                                                 //Label is Rain when we have a rainfall 
 267                                                 if (($pluviolabel = trim($node->div
[1]->div
[0]->div
[0]->div
[1]->p
[1])) == 'Rain') { 
 269                                                         $rainfall = str_replace(' mm', '', $node->div
[1]->div
[0]->div
[0]->div
[1]->p
[1]->span
[0]); 
 270                                                 //Cloud Cover, no rainfall 
 272                                                         //Set rainfall to 0 (mm) 
 277                                                 $data[$zipcode][$dsm][$hour] = [ 
 278                                                         'temperature' => $temperature, 
 279                                                         'realfeel' => $realfeel, 
 280                                                         'rainrisk' => $rainrisk, 
 281                                                         'rainfall' => $rainfall 
 292                 foreach($types as $type => $sessions) { 
 293                         //Iterate on each type 
 294                         foreach($sessions as $sessionId => $session) { 
 296                                 $zipcode = $session->getLocation()->getZipcode(); 
 299                                 $start = $session->getStart(); 
 302                                 if ($type == 'daily') { 
 304                                         $period = [ $start ]; 
 308                                         $stop = $session->getStop(); 
 311                                         $period = new \
DatePeriod( 
 314                                                 //Iterate on each hour 
 315                                                 new \
DateInterval('PT1H'), 
 316                                                 //End with begin + length 
 326                                         'realfeelmin' => null, 
 327                                         'realfeelmax' => null, 
 329                                         'temperaturemin' => null, 
 330                                         'temperaturemax' => null 
 333                                 //Iterate on the period 
 334                                 foreach($period as $time) { 
 336                                         $dsm = $time->format('d/m'); 
 339                                         $hour = $type=='daily'?$type:$time->format('H'); 
 341                                         //Check data availability 
 342                                         //XXX: should never happen 
 343                                         #if (!isset($data[$zipcode][$dsm][$hour])) { 
 344                                         #       //Skip unavailable data 
 349                                         $info = $data[$zipcode][$dsm][$hour]; 
 351                                         //Check if rainrisk is higher 
 352                                         if ($meteo['rainrisk'] === null || $info['rainrisk'] > $meteo['rainrisk']) { 
 353                                                 //Set highest rain risk 
 354                                                 $meteo['rainrisk'] = floatval($info['rainrisk']); 
 357                                         //Check if rainfall is set 
 358                                         if (isset($info['rainfall'])) { 
 360                                                 $meteo['rainfall'] +
= floatval($info['rainfall']); 
 364                                         $meteo['temperature'][$hour] = $info['temperature']; 
 367                                         if ($type != 'daily') { 
 368                                                 //Check min temperature 
 369                                                 if ($meteo['temperaturemin'] === null || $info['temperature'] < $meteo['temperaturemin']) { 
 370                                                         $meteo['temperaturemin'] = floatval($info['temperature']); 
 373                                                 //Check max temperature 
 374                                                 if ($meteo['temperaturemax'] === null || $info['temperature'] > $meteo['temperaturemax']) { 
 375                                                         $meteo['temperaturemax'] = floatval($info['temperature']); 
 379                                         //Check if realfeel is set 
 380                                         if (isset($info['realfeel'])) { 
 382                                                 $meteo['realfeel'][$hour] = $info['realfeel']; 
 385                                                 if ($meteo['realfeelmin'] === null || $info['realfeel'] < $meteo['realfeelmin']) { 
 386                                                         $meteo['realfeelmin'] = floatval($info['realfeel']); 
 390                                                 if ($meteo['realfeelmax'] === null || $info['realfeel'] > $meteo['realfeelmax']) { 
 391                                                         $meteo['realfeelmax'] = floatval($info['realfeel']); 
 396                                 //Check if rainfall is set and differ 
 397                                 if ($session->getRainfall() !== $meteo['rainfall']) { 
 399                                         $session->setRainfall($meteo['rainfall']); 
 402                                 //Check if rainrisk differ 
 403                                 if ($session->getRainrisk() !== $meteo['rainrisk']) { 
 405                                         $session->setRainrisk($meteo['rainrisk']); 
 408                                 //Check realfeel array 
 409                                 if ($meteo['realfeel'] !== []) { 
 411                                         $realfeel = floatval(round(array_sum($meteo['realfeel'])/count($meteo['realfeel']),1)); 
 413                                         //Check if realfeel differ 
 414                                         if ($session->getRealfeel() !== $realfeel) { 
 415                                                 //Set average realfeel 
 416                                                 #$meteo['realfeel'] = array_sum($meteo['realfeel'])/count($meteo['realfeel']); 
 417                                                 $session->setRealfeel($realfeel); 
 420                                         //Check if realfeelmin differ 
 421                                         if ($session->getRealfeelmin() !== $meteo['realfeelmin']) { 
 423                                                 $session->setRealfeelmin($meteo['realfeelmin']); 
 426                                         //Check if realfeelmax differ 
 427                                         if ($session->getRealfeelmax() !== $meteo['realfeelmax']) { 
 429                                                 $session->setRealfeelmax($meteo['realfeelmax']); 
 433                                 //Check temperature array 
 434                                 if ($meteo['temperature'] !== []) { 
 435                                         //Compute temperature 
 436                                         $temperature = floatval(round(array_sum($meteo['temperature'])/count($meteo['temperature']),1)); 
 438                                         //Check if temperature differ 
 439                                         if ($session->getTemperature() !== $temperature) { 
 440                                                 //Set average temperature 
 441                                                 #$meteo['temperature'] = array_sum($meteo['temperature'])/count($meteo['temperature']); 
 442                                                 $session->setTemperature($temperature); 
 445                                         //Check if temperaturemin differ 
 446                                         if ($session->getTemperaturemin() !== $meteo['temperaturemin']) { 
 448                                                 $session->setTemperaturemin($meteo['temperaturemin']); 
 451                                         //Check if temperaturemax differ 
 452                                         if ($session->getTemperaturemax() !== $meteo['temperaturemax']) { 
 454                                                 $session->setTemperaturemax($meteo['temperaturemax']); 
 460                 //Flush to get the ids 
 467                 return self
::SUCCESS
; 
 473          * @return bool|void Return success or exit 
 475         function curl_init() { 
 477                 if (($this->ch 
= curl_init()) === false) { 
 479                         echo 'Curl init failed: '.curl_error($this->ch
)."\n"; 
 490                                         CURLOPT_HTTP_VERSION 
=> CURL_HTTP_VERSION_2_0
, 
 492                                         CURLOPT_HTTPHEADER 
=> [ 
 493                                                 //XXX: it seems that you can disable akamai fucking protection with Pragma: akamai-x-cache-off 
 494                                                 //XXX: see https://support.globaldots.com/hc/en-us/articles/115003996705-Akamai-Pragma-Headers-overview 
 495                                                 #'Pragma: akamai-x-cache-off', 
 496                                                 //XXX: working curl command 
 497                                                 #curl --http2 --cookie file.jar --cookie-jar file.jar -v -i -k -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' -H 'Accept-Language: en-GB,en;q=0.9' -H 'Cache-Control: no-cache' -H 'Connection: keep-alive' -H 'Host: www.accuweather.com' -H 'Pragma: no-cache' -H 'Upgrade-Insecure-Requests: 1' -H 'User-Agent: Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36' 'https://www.accuweather.com/' 
 499                                                 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', 
 500                                                 //Set accept language 
 501                                                 'Accept-Language: en-GB,en;q=0.9', 
 503                                                 'Cache-Control: no-cache', 
 504                                                 //Keep connection alive 
 505                                                 'Connection: keep-alive', 
 508                                                 //Force secure requests 
 509                                                 'Upgrade-Insecure-Requests: 1', 
 511                                                 'User-Agent: Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36', 
 512                                                 //Force akamai cookie 
 513                                                 //XXX: seems to come from http request 
 517                                         CURLOPT_COOKIEFILE 
=> '', 
 518                                         //Disable location following 
 519                                         CURLOPT_FOLLOWLOCATION 
=> false, 
 521                                         #CURLOPT_URL => $url = 'https://www.accuweather.com/', 
 523                                         CURLOPT_HEADER 
=> true, 
 525                                         CURLOPT_RETURNTRANSFER 
=> true, 
 528                                         CURLINFO_HEADER_OUT 
=> true 
 533                         echo 'Curl setopt array failed: '.curl_error($this->ch
)."\n"; 
 545          * @return string|void Return url content or exit 
 547         function curl_get($url) { 
 549                 if (curl_setopt($this->ch
, CURLOPT_URL
, $url) === false) { 
 551                         echo 'Setopt for '.$url.' failed: '.curl_error($this->ch
)."\n"; 
 554                         curl_close($this->ch
); 
 560                 //Check return status 
 561                 if (($response = curl_exec($this->ch
)) === false) { 
 563                         echo 'Get for '.$url.' failed: '.curl_error($this->ch
)."\n"; 
 565                         //Display sent headers 
 566                         var_dump(curl_getinfo($this->ch
, CURLINFO_HEADER_OUT
)); 
 572                         curl_close($this->ch
); 
 579                 if (empty($hs = curl_getinfo($this->ch
, CURLINFO_HEADER_SIZE
))) { 
 581                         echo 'Getinfo for '.$url.' failed: '.curl_error($this->ch
)."\n"; 
 583                         //Display sent headers 
 584                         var_dump(curl_getinfo($this->ch
, CURLINFO_HEADER_OUT
)); 
 590                         curl_close($this->ch
); 
 597                 if (empty($header = substr($response, 0, $hs))) { 
 599                         echo 'Header for '.$url.' empty: '.curl_error($this->ch
)."\n"; 
 601                         //Display sent headers 
 602                         var_dump(curl_getinfo($this->ch
, CURLINFO_HEADER_OUT
)); 
 608                         curl_close($this->ch
); 
 614                 //Check request success 
 615                 if (strlen($header) <= 10 || substr($header, 0, 10) !== 'HTTP/2 200') { 
 617                         echo 'Status for '.$url.' failed: '.curl_error($this->ch
)."\n"; 
 619                         //Display sent headers 
 620                         var_dump(curl_getinfo($this->ch
, CURLINFO_HEADER_OUT
)); 
 626                         curl_close($this->ch
); 
 633                 return substr($response, $hs); 
 639          * @return bool Return success or failure 
 641         function curl_close() { 
 642                 return curl_close($this->ch
);