WordPress.org

Making WordPress.org

Changeset 9100


Ignore:
Timestamp:
08/13/2019 09:08:02 PM (5 weeks ago)
Author:
coreymckrill
Message:

Official WordPress Events: Refactor meetup fetching for upcoming API changes

The Meetup.com API announced changes that will take effect on 2019-08-15. The
API key method of authentication will no longer work, and the entire v2 of the
API will be deactivated in favor of v3. So:

  • Add an OAuth2 client for authentication (copied from WordCamp)
  • Copy updated version of Meetup_Client from WordCamp and refactor usage of its get_events method to reflect the change to v3 endpoints.
  • Add a generic API_Client class (copied from WordCamp) for making repeated remote requests. This is already used by the Meetup_Client class, but now it'll be used for the requests to the WordCamp REST API endpoint and Google Maps as well.
Location:
sites/trunk/wordpress.org/public_html/wp-content/plugins/official-wordpress-events
Files:
2 added
2 edited

Legend:

Unmodified
Added
Removed

  • sites/trunk/wordpress.org/public_html/wp-content/plugins/official-wordpress-events/meetup/class-meetup-client.php

    r9091 r9100  
    11<?php
    2 
    32namespace WordCamp\Utilities;
     3
    44use WP_Error;
    55
    … …  
    99 * Class Meetup_Client
    1010 *
    11  * TODO Refactor this to use the API_Client base class.
     11 * Important: This class and its dependency classes are used in multiple locations in the WordPress/WordCamp
     12 * ecosystem. Because of complexities around SVN externals and the reliability of GitHub's SVN bridge during deploys,
     13 * it was decided to maintain multiple copies of these files rather than have SVN externals pointing to one canonical
     14 * source.
     15 *
     16 * If you make changes to this file, make sure they are propagated to the other locations:
     17 *
     18 * - wordcamp: wp-content/mu-plugins/utilities
     19 * - wporg: wp-content/plugins/official-wordpress-events/meetup
    1220 */
    13 class Meetup_Client {
     21class Meetup_Client extends API_Client {
    1422    /**
    1523     * @var string The base URL for the API endpoints.
    … …  
    1826
    1927    /**
    20      * @var string The API key.
    21      */
    22     protected $api_key = '';
     28     * @var Meetup_OAuth2_Client|null
     29     */
     30    protected $oauth_client = null;
    2331
    2432    /**
    2533     * @var bool If true, the client will fetch fewer results, for faster debugging.
    2634     */
    27     protected $debug_mode;
    28 
    29     /**
    30      * @var WP_Error|null Container for errors.
    31      */
    32     public $error = null;
     35    protected $debug = false;
    3336
    3437    /**
    3538     * Meetup_Client constructor.
    36      */
    37     public function __construct() {
    38         $this->error = new WP_Error();
    39 
    40         if ( defined( 'MEETUP_API_KEY' ) ) {
    41             $this->api_key = MEETUP_API_KEY;
    42         } else {
    43             $this->error->add(
    44                 'api_key_undefined',
    45                 'The Meetup.com API Key is undefined.'
    46             );
    47         }
    48 
    49         $this->debug_mode = apply_filters( 'wcmc_debug_mode', false );
     39     *
     40     * @param array $settings {
     41     *     Optional. Settings for the client.
     42     *
     43     *     @type bool $debug If true, the client will fetch fewer results, for faster debugging.
     44     * }
     45     */
     46    public function __construct( array $settings = [] ) {
     47        parent::__construct( array(
     48            /*
     49             * Response codes that should break the request loop.
     50             *
     51             * See https://www.meetup.com/meetup_api/docs/#errors.
     52             *
     53             * `200` (ok) is not in the list, because it needs to be handled conditionally.
     54             *  See API_Client::tenacious_remote_request.
     55             *
     56             * `400` (bad request) is not in the list, even though it seems like it _should_ indicate an unrecoverable
     57             * error. In practice we've observed that it's common for a seemingly valid request to be rejected with
     58             * a `400` response, but then get a `200` response if that exact same request is retried.
     59             */
     60            'breaking_response_codes' => array(
     61                401, // Unauthorized (invalid key).
     62                429, // Too many requests (rate-limited).
     63                404, // Unable to find group
     64            ),
     65            'throttle_callback'       => array( __CLASS__, 'throttle' ),
     66        ) );
     67
     68        $settings = wp_parse_args(
     69            $settings,
     70            array(
     71                'debug' => false,
     72            )
     73        );
     74
     75        $this->oauth_client = new Meetup_OAuth2_Client;
     76        $this->debug        = $settings['debug'];
     77
     78        if ( $this->debug ) {
     79            self::cli_message( "Meetup Client debug is ON. Results will be truncated." );
     80        }
    5081    }
    5182
    … …  
    6899
    69100        while ( $request_url ) {
    70             $request_url = $this->sign_request_url( $request_url );
    71 
    72             $response = $this->tenacious_remote_get( $request_url );
     101            $response = $this->tenacious_remote_get( $request_url, $this->get_request_args() );
    73102
    74103            if ( 200 === wp_remote_retrieve_response_code( $response ) ) {
    … …  
    98127            }
    99128
    100             if ( $this->debug_mode ) {
     129            if ( $this->debug ) {
    101130                break;
    102131            }
    … …  
    125154        ), $request_url );
    126155
    127         $request_url = $this->sign_request_url( $request_url );
    128 
    129         $response = $this->tenacious_remote_get( $request_url );
     156        $response = $this->tenacious_remote_get( $request_url, $this->get_request_args() );
    130157
    131158        if ( 200 === wp_remote_retrieve_response_code( $response ) ) {
    … …  
    152179
    153180    /**
    154      * Wrapper for `wp_remote_get` to retry requests that fail temporarily for various reasons.
    155      *
    156      * One common example of a reason a request would fail, but later succeed, is when the first request times out.
    157      *
    158      * Based on `wcorg_redundant_remote_get`.
    159      *
    160      * @param string $url
    161      * @param array  $args
    162      *
    163      * @return array|WP_Error
    164      */
    165     protected function tenacious_remote_get( $url, $args = array() ) {
    166         $attempt_count = 0;
    167         $max_attempts  = 3;
    168 
    169         /*
    170          * Response codes that should break the loop.
    171          *
    172          * See https://www.meetup.com/meetup_api/docs/#errors.
    173          *
    174          * `200` (ok) is not in the list, because it needs to be handled conditionally. See below.
    175          *
    176          * `400` (bad request) is not in the list, even though it seems like it _should_ indicate an unrecoverable
    177          * error. In practice we've observed that it's common for a seemingly valid request to be rejected with
    178          * a `400` response, but then get a `200` response if that exact same request is retried.
    179          */
    180         $breaking_codes = array(
    181             401, // Unauthorized (invalid key).
    182             429, // Too many requests (rate-limited).
    183             404, // Unable to find group
     181     * Generate headers to use in a request.
     182     *
     183     * @return array
     184     */
     185    protected function get_request_args() {
     186        $oauth_token = $this->oauth_client->get_oauth_token();
     187
     188        return array(
     189            'headers' => array(
     190                'Accept'        => 'application/json',
     191                'Authorization' => "Bearer $oauth_token",
     192            ),
    184193        );
    185 
    186         // The default of 5 seconds often results in frequent timeouts.
    187         if ( empty( $args['timeout'] ) ) {
    188             $args['timeout'] = 15;
    189         }
    190 
    191         while ( $attempt_count < $max_attempts ) {
    192             $response      = wp_remote_get( $url, $args );
    193             $response_code = wp_remote_retrieve_response_code( $response );
    194 
    195             $this->maybe_throttle( wp_remote_retrieve_headers( $response ) );
    196 
    197             /*
    198              * Sometimes their API inexplicably returns a success code with an empty body, but will return a valid
    199              * response if the exact request is retried.
    200              */
    201             if ( 200 === $response_code && ! empty( wp_remote_retrieve_body( $response ) ) ) {
    202                 break;
    203             }
    204 
    205             if ( in_array( $response_code, $breaking_codes, true ) ) {
    206                 break;
    207             }
    208 
    209             $attempt_count++;
    210 
    211             /**
    212              * Action: Fires when tenacious_remote_get fails a request attempt.
    213              *
    214              * Note that the request parameter includes the request URL that contains a query string for the API key.
    215              * This should be redacted before outputting anywhere public.
    216              *
    217              * @param array $response
    218              * @param array $request
    219              * @param int   $attempt_count
    220              * @param int   $max_attempts
    221              */
    222             do_action( 'meetup_client_tenacious_remote_get_attempt', $response, compact( 'url', 'args' ), $attempt_count, $max_attempts );
    223 
    224             if ( $attempt_count < $max_attempts ) {
    225                 $retry_after = wp_remote_retrieve_header( $response, 'retry-after' ) ?: 5;
    226                 $wait        = min( $retry_after * $attempt_count, 30 );
    227 
    228                 if ( 'cli' === php_sapi_name() ) {
    229                     echo "\nRequest failed $attempt_count times. Pausing for $wait seconds before retrying.";
    230                 }
    231 
    232                 sleep( $wait );
    233             }
    234         }
    235 
    236         if ( $attempt_count === $max_attempts && 'cli' === php_sapi_name() ) {
    237             if ( 200 !== $response_code || is_wp_error( $response ) ) {
    238                 echo "\nRequest failed $attempt_count times. Giving up.";
    239             }
    240         }
    241 
    242         return $response;
    243     }
    244 
    245     /**
    246      * Sign a request URL with our API key.
    247      *
    248      * @param string $request_url
    249      *
    250      * @return string
    251      */
    252     protected function sign_request_url( $request_url ) {
    253         return add_query_arg( array(
    254             'sign' => true,
    255             'key'  => $this->api_key,
    256         ), $request_url );
    257194    }
    258195
    … …  
    298235     *
    299236     * @param array $headers
    300      */
    301     protected function maybe_throttle( $headers ) {
     237     *
     238     * @return void
     239     */
     240    protected static function throttle( $response ) {
     241        $headers = wp_remote_retrieve_headers( $response );
     242
    302243        if ( ! isset( $headers['x-ratelimit-remaining'], $headers['x-ratelimit-reset'] ) ) {
    303244            return;
    … …  
    305246
    306247        $remaining = absint( $headers['x-ratelimit-remaining'] );
    307         $period    = absint( $headers['x-ratelimit-reset'    ] );
    308 
    309         // Pause more frequently than we need to, and for longer, just to be safe.
    310         if ( $remaining > 2 ) {
     248        $period    = absint( $headers['x-ratelimit-reset'] );
     249
     250        /**
     251         * Don't throttle if we have sufficient requests remaining.
     252         *
     253         * We don't let this number get to 0, though, because there are scenarios where multiple processes are using
     254         * the API at the same time, and there's no way for them to be aware of each other.
     255         */
     256        if ( $remaining > 3 ) {
    311257            return;
    312258        }
    313259
     260        // Pause for longer than we need to, just to be safe.
    314261        if ( $period < 2 ) {
    315262            $period = 2;
    316263        }
    317264
    318         if ( 'cli' === php_sapi_name() ) {
    319             echo "\nPausing for $period seconds to avoid rate-limiting.";
    320         }
     265        self::cli_message( "\nPausing for $period seconds to avoid rate-limiting." );
    321266
    322267        sleep( $period );
    … …  
    326271     * Extract error information from an API response and add it to our error handler.
    327272     *
     273     * Make sure you don't include the full $response in the error as data, as that could expose sensitive information
     274     * from the request payload.
     275     *
    328276     * @param array|WP_Error $response
    329277     *
    330278     * @return void
    331279     */
    332     protected function handle_error_response( $response ) {
    333         if ( is_wp_error( $response ) ) {
    334             $codes = $response->get_error_codes();
    335 
    336             foreach ( $codes as $code ) {
    337                 $messages = $response->get_error_messages( $code );
    338 
    339                 foreach ( $messages as $message ) {
    340                     $this->error->add( $code, $message );
    341                 }
    342             }
    343 
     280    public function handle_error_response( $response ) {
     281        if ( parent::handle_error_response( $response ) ) {
    344282            return;
    345283        }
    … …  
    350288        if ( isset( $data['errors'] ) ) {
    351289            foreach ( $data['errors'] as $error ) {
    352                 $this->error->add( $error['code'], $error['message'] );
     290                $this->error->add(
     291                    $error['code'],
     292                    $error['message']
     293                );
    353294            }
    354295        } elseif ( isset( $data['code'] ) && isset( $data['details'] ) ) {
    355             $this->error->add( $data['code'], $data['details'] );
     296            $this->error->add(
     297                $data['code'],
     298                $data['details']
     299            );
    356300        } elseif ( $response_code ) {
    357301            $this->error->add(
    … …  
    360304            );
    361305        } else {
    362             $this->error->add( 'unknown_error', 'There was an unknown error.' );
     306            $this->error->add(
     307                'unknown_error',
     308                'There was an unknown error.'
     309            );
    363310        }
    364311    }
    … …  
    385332     * Retrieve data about events associated with a set of groups.
    386333     *
    387      * This automatically breaks up requests into chunks of 50 groups to avoid overloading the API.
    388      *
    389      * @param array $group_ids The IDs of the groups to get events for.
    390      * @param array $args      Optional. Additional request parameters.
    391      *                         See https://www.meetup.com/meetup_api/docs/2/events/.
     334     * Because of the way that the Meetup API v3 endpoints are structured, we unfortunately have to make one request
     335     * (or more, if there's pagination) for each group that we want events for. When there are hundreds of groups, and
     336     * we are throttling to make sure we don't get rate-limited, this process can literally take several minutes.
     337     *
     338     * So, when building the array for the $group_slugs parameter, it's important to filter out groups that you know
     339     * will not provide relevant results. For example, if you want all events during a date range in the past, you can
     340     * filter out groups that didn't join the chapter program until after your date range.
     341     *
     342     * Note that when using date/time related parameters in the $args array, unlike other endpoints and fields in the
     343     * Meetup API which use an epoch timestamp in milliseconds, this one requires a date/time string formatted in
     344     * ISO 8601, without the timezone part. Because consistency is overrated.
     345     *
     346     * @param array $group_slugs The URL slugs of each group to retrieve events for. Also known as `urlname`.
     347     * @param array $args        Optional. Additional request parameters.
     348     *                           See https://www.meetup.com/meetup_api/docs/:urlname/events/#list
    392349     *
    393350     * @return array|WP_Error
    394351     */
    395     public function get_events( array $group_ids, array $args = array() ) {
    396         $url_base     = $this->api_base . '2/events';
    397         $group_chunks = array_chunk( $group_ids, 50, true ); // Meetup API sometimes throws an error with chunk size larger than 50.
    398         $events       = array();
    399 
    400         foreach ( $group_chunks as $chunk ) {
    401             $query_args = array_merge( array(
    402                 'group_id' => implode( ',', $chunk ),
    403             ), $args );
    404 
    405             $request_url = add_query_arg( $query_args, $url_base );
    406 
    407             $data = $this->send_paginated_request( $request_url );
    408 
    409             if ( is_wp_error( $data ) ) {
    410                 return $data;
    411             }
    412 
    413             $events = array_merge( $events, $data );
     352    public function get_events( array $group_slugs, array $args = array() ) {
     353        $events = array();
     354
     355        if ( $this->debug ) {
     356            $chunked     = array_chunk( $group_slugs, 10 );
     357            $group_slugs = $chunked[0];
     358        }
     359
     360        foreach ( $group_slugs as $group_slug ) {
     361            $response = $this->get_group_events( $group_slug, $args );
     362
     363            if ( is_wp_error( $response ) ) {
     364                return $response;
     365            }
     366
     367            $events = array_merge( $events, $response );
    414368        }
    415369
    … …  
    418372
    419373    /**
    420      * Retrieve data about the group. Calls https://www.meetup.com/meetup_api/docs/:urlname/#get
     374     * Retrieve details about a group.
    421375     *
    422376     * @param string $group_slug The slug/urlname of a group.
    423      * @param array $args Optional. Additional request parameters.
     377     * @param array  $args       Optional. Additional request parameters.
     378     *                           See https://www.meetup.com/meetup_api/docs/:urlname/#get
    424379     *
    425380     * @return array|WP_Error
    426381     */
    427     public function get_group_details ( $group_slug, $args = array() ) {
    428         $request_url = $this->api_base . "$group_slug";
     382    public function get_group_details( $group_slug, $args = array() ) {
     383        $request_url = $this->api_base . sanitize_key( $group_slug );
    429384
    430385        if ( ! empty( $args ) ) {
    … …  
    436391
    437392    /**
    438      * Retrieve group members. Calls https://www.meetup.com/meetup_api/docs/:urlname/members/#list
     393     * Retrieve details about group members.
    439394     *
    440395     * @param string $group_slug The slug/urlname of a group.
    441      * @param array $args Optional. Additional request parameters.
     396     * @param array  $args       Optional. Additional request parameters.
     397     *                           See https://www.meetup.com/meetup_api/docs/:urlname/members/#list
    442398     *
    443399     * @return array|WP_Error
    444400     */
    445     public function get_group_members ( $group_slug, $args = array() ) {
    446         $request_url = $this->api_base . "$group_slug/members";
     401    public function get_group_members( $group_slug, $args = array() ) {
     402        $request_url = $this->api_base . sanitize_key( $group_slug ) . '/members';
    447403
    448404        if ( ! empty( $args ) ) {
    … …  
    458414     * @param string $group_slug The slug/urlname of a group.
    459415     * @param array  $args       Optional. Additional request parameters.
    460      *                           See https://www.meetup.com/meetup_api/docs/:urlname/events/.
     416     *                           See https://www.meetup.com/meetup_api/docs/:urlname/events/#list
    461417     *
    462418     * @return array|WP_Error
    463419     */
    464420    public function get_group_events( $group_slug, array $args = array() ) {
    465         $request_url = $this->api_base . "$group_slug/events";
     421        $request_url = $this->api_base . sanitize_key( $group_slug ) . '/events';
    466422
    467423        if ( ! empty( $args ) ) {

  • sites/trunk/wordpress.org/public_html/wp-content/plugins/official-wordpress-events/official-wordpress-events.php

    r9091 r9100  
    88*/
    99
     10use WordCamp\Utilities\API_Client;
    1011use WordCamp\Utilities\Meetup_Client;
    1112
    … …  
    3839     */
    3940    public function __construct() {
    40         add_action( 'wp_enqueue_scripts',           array( $this, 'enqueue_scripts'    ) );
    41         add_action( 'owpe_prime_events_cache',      array( $this, 'prime_events_cache' ) );
    42         add_action( 'owpe_mark_deleted_meetups',    array( $this, 'mark_deleted_meetups' ) );
     41        add_action( 'wp_enqueue_scripts',               array( $this, 'enqueue_scripts'       ) );
     42        add_action( 'owpe_prime_events_cache',          array( $this, 'prime_events_cache'    ) );
     43        add_action( 'owpe_mark_deleted_meetups',        array( $this, 'mark_deleted_meetups'  ) );
     44        add_action( 'api_client_handle_error_response', array( $this, 'handle_error_response' ), 10, 3 );
     45
    4346        add_shortcode( 'official_wordpress_events', array( $this, 'render_events'      ) );
    4447
    … …  
    5053            wp_schedule_event( time(), 'hourly', 'owpe_mark_deleted_meetups' );
    5154        }
     55    }
     56
     57    /**
     58     * Get an instance of the generic API Client, loading files first as necessary.
     59     *
     60     * @return API_Client
     61     */
     62    protected function get_api_client() {
     63        if ( ! class_exists( '\WordCamp\Utilities\API_Client' ) ) {
     64            $files = array(
     65                'class-api-client.php',
     66            );
     67
     68            foreach ( $files as $file ) {
     69                require_once trailingslashit( __DIR__ ) . "meetup/$file";
     70            }
     71        }
     72
     73        return new API_Client();
    5274    }
    5375
    … …  
    6082        if ( ! class_exists( '\WordCamp\Utilities\Meetup_Client' ) ) {
    6183            $files = array(
     84                'class-api-client.php',
     85                'class-meetup-oauth2-client.php',
    6286                'class-meetup-client.php',
    6387            );
    … …  
    209233     */
    210234    protected function fetch_upcoming_events() {
    211         $events = array_merge( $this->get_wordcamp_events(), $this->get_meetup_events() );
     235        $wordcamp_events = $this->get_wordcamp_events();
     236        $meetup_events   = $this->get_meetup_events();
     237
     238        $events = array_merge( $wordcamp_events, $meetup_events );
    212239
    213240        return $events;
    … …  
    239266     */
    240267    protected function get_wordcamp_events() {
    241         $request_params = array(
     268        $api_client = $this->get_api_client();
     269
     270        // Note: With the number of WordCamps per year growing fast, we may need to batch requests in the near future, like we do for meetups.
     271        $request_url = add_query_arg( array(
    242272            'status'   => array( 'wcpt-scheduled', 'wcpt-cancelled' ),
    243273            'per_page' => 100,
    244             // Note: With the number of WordCamps per year growing fast, we may need to batch requests in the near future, like we do for meetups
    245         );
    246 
    247         $endpoint = add_query_arg( $request_params, self::WORDCAMP_API_BASE_URL . 'wp/v2/wordcamps' );
    248         $response = $this->remote_get( esc_url_raw( $endpoint ) );
    249         $events   = $this->parse_wordcamp_events( $response );
     274        ), self::WORDCAMP_API_BASE_URL . 'wp/v2/wordcamps' );
     275
     276        $response = $api_client->tenacious_remote_get( $request_url );
     277
     278        $api_client->handle_error_response( $response, $request_url );
     279
     280        $events = $this->parse_wordcamp_events( $response );
    250281
    251282        $this->log( sprintf( 'returning %d events', count( $events ) ) );
    … …  
    339370        $events = array();
    340371
    341         $client = $this->get_meetup_client();
    342         if ( ! empty( $client->error->errors ) ) {
    343             $this->log( 'Failed to instantiate meetup client: ' . wp_json_encode( $client->error ), true );
     372        // Fetching events for a large number of groups from the Meetup API is currently a very inefficient process.
     373        ini_set( 'memory_limit', '900M' );
     374        ini_set( 'max_execution_time', 500 );
     375
     376        $meetup_client = $this->get_meetup_client();
     377        if ( ! empty( $meetup_client->error->errors ) ) {
     378            $this->log( 'Failed to instantiate meetup client: ' . wp_json_encode( $meetup_client->error ), true );
    344379            return $events;
    345380        }
    346381
    347         $groups = $client->get_groups();
    348         if ( ! empty( $client->error->errors ) ) {
    349             $this->log( 'Failed to fetch groups: ' . wp_json_encode( $client->error ), true );
     382        $groups = $meetup_client->get_groups();
     383        if ( ! empty( $meetup_client->error->errors ) ) {
     384            $this->log( 'Failed to fetch groups: ' . wp_json_encode( $meetup_client->error ), true );
    350385            return $events;
    351386        }
    352387
    353         $meetups = $client->get_events( wp_list_pluck( $groups, 'id' ) );
    354         if ( ! empty( $client->error->errors ) ) {
    355             $this->log( 'Failed to fetch meetups: ' . wp_json_encode( $client->error ), true );
     388        $yesterday    = date( 'c', strtotime( '-1 day' ) );
     389        $one_year_out = date( 'c', strtotime( '+1 year' ) );
     390        $meetups      = $meetup_client->get_events(
     391            wp_list_pluck( $groups, 'urlname' ),
     392            array(
     393                // We want cancelled events too so they will be updated in our database table.
     394                'status'          => 'upcoming,cancelled',
     395                // We don't want cancelled events in the past, but need some leeway here for timezones.
     396                'no_earlier_than' => substr( $yesterday, 0, strpos( $yesterday, '+' ) ),
     397                // We don't need to cache events happening more than a year from now.
     398                'no_later_than'   => substr( $one_year_out, 0, strpos( $one_year_out, '+' ) ),
     399            )
     400        );
     401        if ( ! empty( $meetup_client->error->errors ) ) {
     402            $this->log( 'Failed to fetch meetups: ' . wp_json_encode( $meetup_client->error ), true );
    356403            return $events;
    357404        }
    358405
    359                     foreach ( $meetups as $meetup ) {
    360                         if ( empty( $meetup['id'] ) || empty( $meetup['name'] ) ) {
    361                             $this->log( 'Malformed meetup: ' . wp_json_encode( $meetup ) );
    362                             continue;
    363                         }
    364 
    365                         $start_timestamp = ( $meetup['time'] / 1000 ) + ( $meetup['utc_offset'] / 1000 );    // convert to seconds
    366 
    367                         if ( isset( $meetup['venue'] ) ) {
    368                             $location = $this->format_meetup_venue_location( $meetup['venue'] );
    369                         } else {
    370                             $geocoded_location = $this->reverse_geocode( $meetup['group']['group_lat'], $meetup['group']['group_lon'] );
    371                             $location_parts    = $this->parse_reverse_geocode_address( $geocoded_location );
    372                             $location          = sprintf(
    373                                 '%s%s%s',
    374                                 $location_parts['city'] ?? '',
    375                                 empty( $location_parts['state'] )        ? '' : ', ' . $location_parts['state'],
    376                                 empty( $location_parts['country_name'] ) ? '' : ', ' . $location_parts['country_name']
    377                             );
    378                             $location         = trim( $location, ", \t\n\r\0\x0B" );
    379                         }
    380 
    381                         if ( ! empty( $meetup['venue']['country'] ) ) {
    382                             $country_code = $meetup['venue']['country'];
    383                         } elseif ( ! empty( $location_parts['country_code'] ) ) {
    384                             $country_code = $location_parts['country_code'];
    385                         } else {
    386                             $country_code = '';
    387                         }
    388 
    389                         $events[] = new Official_WordPress_Event( array(
    390                             'type'            => 'meetup',
    391                             'source_id'       => $meetup['id'],
    392                             'status'          => 'upcoming' === $meetup['status'] ? 'scheduled' : 'cancelled',
    393                             'title'           => $meetup['name'],
    394                             'url'             => $meetup['event_url'],
    395                             'meetup_name'     => $meetup['group']['name'],
    396                             'meetup_url'      => sprintf( 'https://www.meetup.com/%s/', $meetup['group']['urlname'] ),
    397                             'description'     => $meetup['description'] ?? '',
    398                             'num_attendees'   => $meetup['yes_rsvp_count'],
    399                             'start_timestamp' => $start_timestamp,
    400                             'end_timestamp'   => ( empty ( $meetup['duration'] ) ? $start_timestamp : $start_timestamp + ( $meetup['duration'] / 1000 ) ), // convert to seconds
    401                             'location'        => $location,
    402                             'country_code'    => $country_code,
    403                             'latitude'        => empty( $meetup['venue']['lat'] ) ? $meetup['group']['group_lat'] : $meetup['venue']['lat'],
    404                             'longitude'       => empty( $meetup['venue']['lon'] ) ? $meetup['group']['group_lon'] : $meetup['venue']['lon'],
    405                         ) );
    406                     }
     406        $events = $this->parse_meetup_events( $meetups );
    407407
    408408        $this->log( sprintf( 'returning %d events', count( $events ) ) );
     409
     410        return $events;
     411    }
     412
     413    /**
     414     * Parse meetup events out of a response from the Meetup API.
     415     *
     416     * @param array $meetups
     417     *
     418     * @return array
     419     */
     420    protected function parse_meetup_events( $meetups ) {
     421        $events = array();
     422
     423        foreach ( $meetups as $meetup ) {
     424            if ( empty( $meetup['id'] ) || empty( $meetup['name'] ) ) {
     425                $this->log( 'Malformed meetup: ' . wp_json_encode( $meetup ) );
     426                continue;
     427            }
     428
     429            $start_timestamp = ( $meetup['time'] / 1000 ) + ( $meetup['utc_offset'] / 1000 ); // convert to seconds
     430            $latitude        = ! empty( $meetup['venue']['lat'] ) ? $meetup['venue']['lat'] : $meetup['group']['lat'];
     431            $longitude       = ! empty( $meetup['venue']['lon'] ) ? $meetup['venue']['lon'] : $meetup['group']['lon'];
     432
     433            if ( isset( $meetup['venue'] ) ) {
     434                $location = $this->format_meetup_venue_location( $meetup['venue'] );
     435            } else {
     436                $geocoded_location = $this->reverse_geocode( $latitude, $longitude );
     437                $location_parts    = $this->parse_reverse_geocode_address( $geocoded_location );
     438                $location          = sprintf(
     439                    '%s%s%s',
     440                    $location_parts['city'] ?? '',
     441                    empty( $location_parts['state'] )        ? '' : ', ' . $location_parts['state'],
     442                    empty( $location_parts['country_name'] ) ? '' : ', ' . $location_parts['country_name']
     443                );
     444                $location          = trim( $location, ", \t\n\r\0\x0B" );
     445            }
     446
     447            if ( ! empty( $meetup['venue']['country'] ) ) {
     448                $country_code = $meetup['venue']['country'];
     449            } elseif ( ! empty( $location_parts['country_code'] ) ) {
     450                $country_code = $location_parts['country_code'];
     451            } else {
     452                $country_code = '';
     453            }
     454
     455            $events[] = new Official_WordPress_Event( array(
     456                'type'            => 'meetup',
     457                'source_id'       => $meetup['id'],
     458                'status'          => 'upcoming' === $meetup['status'] ? 'scheduled' : 'cancelled',
     459                'title'           => $meetup['name'],
     460                'url'             => $meetup['link'],
     461                'meetup_name'     => $meetup['group']['name'],
     462                'meetup_url'      => sprintf( 'https://www.meetup.com/%s/', $meetup['group']['urlname'] ),
     463                'description'     => $meetup['description'] ?? '',
     464                'num_attendees'   => $meetup['yes_rsvp_count'],
     465                'start_timestamp' => $start_timestamp,
     466                'end_timestamp'   => ( empty ( $meetup['duration'] ) ? $start_timestamp : $start_timestamp + ( $meetup['duration'] / 1000 ) ), // convert to seconds
     467                'location'        => $location,
     468                'country_code'    => $country_code,
     469                'latitude'        => $latitude,
     470                'longitude'       => $longitude,
     471            ) );
     472        }
    409473
    410474        return $events;
    … …  
    431495        usleep( 75000 );
    432496
    433         $response = $this->remote_get( sprintf(
     497        $api_client  = $this->get_api_client();
     498        $request_url = sprintf(
    434499            'https://maps.googleapis.com/maps/api/geocode/json?latlng=%s,%s&sensor=false&key=%s',
    435500            $latitude,
    436501            $longitude,
    437502            OFFICIAL_WP_EVENTS_GOOGLE_MAPS_API_KEY
    438         ) );
    439         $body = json_decode( wp_remote_retrieve_body( $response ) );
     503        );
     504
     505        $response = $api_client->tenacious_remote_get( $request_url );
     506        $body     = json_decode( wp_remote_retrieve_body( $response ) );
    440507
    441508        if ( ! is_wp_error( $response ) && isset( $body->results ) && empty( $body->error_message ) ) {
    … …  
    448515        }
    449516        else {
     517            $api_client->handle_error_response( $response, $request_url );
    450518            $this->log( 'geocode failed: ' . wp_json_encode( $response ) );
    451519        }
    … …  
    571639                $wpdb->update( self::EVENTS_TABLE, array( 'status' => 'deleted' ), array( 'id' => $db_event->id ) );
    572640
    573                 if ( 'cli' === php_sapi_name() ) {
    574                     echo "\nMarked {$db_event->source_id} as deleted.";
    575                 }
    576             }
    577         }
    578     }
    579 
    580     /**
    581      * Wrapper for wp_remote_get()
    582      *
    583      * This adds error logging/notification.
    584      *
    585      * @param string $url
    586      * @param array  $args
    587      *
    588      * @return false|array|WP_Error False if a valid $url was not passed; otherwise the results from wp_remote_get()
    589      */
    590     protected function remote_get( $url, $args = array() ) {
    591         $response = $error = false;
    592 
    593         if ( $url ) {
    594             $args['timeout'] = 30;
    595             $response        = wp_remote_get( $url, $args );
    596 
    597             $this->maybe_pause( wp_remote_retrieve_headers( $response ) );
    598 
    599             $response_code    = wp_remote_retrieve_response_code( $response );
    600             $response_message = wp_remote_retrieve_response_message( $response );
    601             $response_body    = wp_remote_retrieve_body( $response );
    602 
    603             if ( is_wp_error( $response ) ) {
    604                 $error_messages = implode( ', ', $response->get_error_messages() );
    605 
    606                 if ( false === strpos( $error_messages, 'Operation timed out' ) ) {
    607                     $error = sprintf(
    608                         'Received WP_Error message: %s; Request was to %s; Arguments were: %s',
    609                         $error_messages,
    610                         $url,
    611                         print_r( $args, true )
    612                     );
    613                 }
    614             } elseif ( 200 != $response_code ) {
    615                 // trigger_error() has a message limit of 1024 bytes, so we truncate $response['body'] to make sure that $body doesn't get truncated.
     641                $this->log( "Marked {$db_event->source_id} as deleted." );
     642            }
     643        }
     644    }
     645
     646    /**
     647     * Error logging and notification.
     648     *
     649     * Hooked to `api_client_handle_error_response`.
     650     *
     651     * @param array|WP_Error $response
     652     *
     653     * @return void
     654     */
     655    protected function handle_error_response( $response, $request_url, $request_args ) {
     656        $error = null;
     657
     658        $response_code    = wp_remote_retrieve_response_code( $response );
     659        $response_message = wp_remote_retrieve_response_message( $response );
     660        $response_body    = wp_remote_retrieve_body( $response );
     661
     662        if ( is_wp_error( $response ) ) {
     663            $error_messages = implode( ', ', $response->get_error_messages() );
     664
     665            if ( false === strpos( $error_messages, 'Operation timed out' ) ) {
    616666                $error = sprintf(
    617                     "HTTP Code: %d\nMessage: %s\nBody: %s\nRequest URL: %s\nArgs: %s",
    618                     $response_code,
    619                     sanitize_text_field( $response_message ),
    620                     substr( sanitize_text_field( $response_body ), 0, 500 ),
    621                     $url,
    622                     print_r( $args, true )
     667                    'Received WP_Error message: %s; Request was to %s; Arguments were: %s',
     668                    $error_messages,
     669                    $request_url,
     670                    print_r( $request_args, true )
    623671                );
    624 
    625                 $response = new WP_Error( 'owe_invalid_http_response', 'Invalid HTTP response code', $response );
    626             }
    627 
    628             if ( $error ) {
    629                 $error = preg_replace( '/&key=[a-z0-9]+/i', '&key=[redacted]', $error );
    630                 trigger_error( sprintf(
    631                     '%s error for %s: %s',
    632                     __METHOD__,
    633                     parse_url( site_url(), PHP_URL_HOST ),
     672            }
     673        } elseif ( 200 !== $response_code ) {
     674            // trigger_error() has a message limit of 1024 bytes, so we truncate $response['body'] to make sure that $body doesn't get truncated.
     675            $error = sprintf(
     676                "HTTP Code: %d\nMessage: %s\nBody: %s\nRequest URL: %s\nArgs: %s",
     677                $response_code,
     678                sanitize_text_field( $response_message ),
     679                substr( sanitize_text_field( $response_body ), 0, 500 ),
     680                $request_url,
     681                print_r( $request_args, true )
     682            );
     683        }
     684
     685        if ( $error ) {
     686            $error = preg_replace( '/&key=[a-z0-9]+/i', '&key=[redacted]', $error );
     687
     688            $this->log( sanitize_text_field( $error ), true );
     689
     690            trigger_error( sprintf(
     691                '%s error for %s: %s',
     692                __METHOD__,
     693                parse_url( site_url(), PHP_URL_HOST ),
     694                sanitize_text_field( $error )
     695            ), E_USER_WARNING );
     696
     697            $to = apply_filters( 'owe_error_email_addresses', array() );
     698
     699            if ( $to && ( ! defined( 'WPORG_SANDBOXED_REQUEST' ) || ! WPORG_SANDBOXED_REQUEST ) ) {
     700                wp_mail(
     701                    $to,
     702                    sprintf(
     703                        '%s error for %s',
     704                        __METHOD__,
     705                        parse_url( site_url(), PHP_URL_HOST )
     706                    ),
    634707                    sanitize_text_field( $error )
    635                 ), E_USER_WARNING );
    636 
    637                 $to = apply_filters( 'owe_error_email_addresses', array() );
    638                 if ( $to && ( ! defined( 'WPORG_SANDBOXED_REQUEST' ) || ! WPORG_SANDBOXED_REQUEST ) ) {
    639                     wp_mail(
    640                         $to,
    641                         sprintf(
    642                             '%s error for %s',
    643                             __METHOD__,
    644                             parse_url( site_url(), PHP_URL_HOST )
    645                         ),
    646                         sanitize_text_field( $error )
    647                     );
    648                 }
    649             }
    650         }
    651 
    652         return $response;
    653     }
    654 
    655     /**
    656      * Maybe pause the script to avoid rate limiting
    657      *
    658      * @param array $headers
    659      */
    660     protected function maybe_pause( $headers ) {
    661         if ( ! isset( $headers['x-ratelimit-remaining'], $headers['x-ratelimit-reset'] ) ) {
    662             return;
    663         }
    664 
    665         $remaining = absint( $headers['x-ratelimit-remaining'] );
    666         $period    = absint( $headers['x-ratelimit-reset'] );
    667 
    668         // Pause more frequently than we need to, and for longer, just to be safe
    669         if ( $remaining > 2 ) {
    670             return;
    671         }
    672 
    673         if ( $period < 2 ) {
    674             $period = 2;
    675         }
    676 
    677         if ( 'cli' == php_sapi_name() ) {
    678             echo "\nPausing for $period seconds to avoid rate-limiting.";
    679         }
    680 
    681         $this->log( 'sleeping to avoid api rate limit' );
    682         sleep( $period );
     708                );
     709            }
     710        }
    683711    }
    684712
    … …  
    697725        $limit = 500;
    698726        $api_keys = array( MEETUP_API_KEY, OFFICIAL_WP_EVENTS_GOOGLE_MAPS_API_KEY );
     727
     728        if ( 'cli' === php_sapi_name() ) {
     729            echo $message;
     730        }
    699731
    700732        if ( $write_to_disk ) {
Note: See TracChangeset for help on using the changeset viewer.