WordPress.org

Making WordPress.org

Changeset 5169


Ignore:
Timestamp:
03/27/2017 10:22:40 AM (3 years ago)
Author:
tellyworth
Message:

Plugin directory search: randomly throttle back searches for a short window if the API starts returning errors or failing to respond.

This should help limit the load on the ES API. Constants for the window and the threshold curve might need tweaking.

See also [5124].

File:
1 edited

Legend:

Unmodified
Added
Removed
  • sites/trunk/wordpress.org/public_html/wp-content/plugins/plugin-directory/libs/site-search/jetpack-search.php

    r5125 r5169  
    7373    const CACHE_GROUP = 'jetpack-search';
    7474    const CACHE_EXPIRY = 300;
     75    const ERROR_COUNT_KEY = 'error-count-';
     76    const ERROR_COUNT_WINDOW = 60; // seconds
    7577
    7678    protected function __construct() {
     
    167169    /////////////////////////////////////////////////////////
    168170    // Raw Search Query
     171
     172    /*
     173     * Return a count of the number of search API errors within the last ERROR_COUNT_WINDOW seconds
     174     */
     175    protected function get_error_volume() {
     176        // Use a dual-tick window like nonces
     177        $tick = ceil( time() / (self::ERROR_COUNT_WINDOW/2) );
     178
     179        return intval( wp_cache_get( self::ERROR_COUNT_KEY . $tick, self::CACHE_GROUP ) )
     180             + intval( wp_cache_get( self::ERROR_COUNT_KEY . ($tick -1), self::CACHE_GROUP ) );
     181    }
     182
     183    /*
     184     *  Increment the recent error volume by $count.
     185     */
     186    protected function increment_error_volume( $count = 1 ) {
     187        // wp_cache_incr() bails if the key does not exist
     188        $tick = ceil( time() / (self::ERROR_COUNT_WINDOW/2) );
     189        wp_cache_add( self::ERROR_COUNT_KEY . $tick, 0, self::CACHE_GROUP, self::ERROR_COUNT_WINDOW );
     190        return wp_cache_incr( self::ERROR_COUNT_KEY . $tick, $count, self::CACHE_GROUP );
     191    }
     192
     193    /*
     194     * Determine whether or not the recent error volume is low enough to allow a fresh API call.
     195     */
     196    protected function error_volume_is_low() {
     197        // This cache key keeps a global-ish count of recent errors (per memcache instance).
     198        // Used for random exponential backoff when errors start to pile up, as they will if there is a network outage for example.
     199        $error_volume = max( $this->get_error_volume(), 0 ); // >= 0
     200
     201        // This gives us a threshold with a gentle curve from 10 down to 0
     202        // The idea being that for a small volume of errors ( < 10 ) we'll have a 100% chance of attempting a new search
     203        // For 10-15 errors, an 80-90% chance
     204        // For 20 errors, a 50% chance
     205        // For 40+ errors, a 0% chance
     206
     207        $threshold = ceil( 10 / ( 1 + pow( $error_volume / 20, 4 ) ) );
     208        return mt_rand( 1, 10 ) <= $threshold;
     209    }
     210
     211    /*
     212     * Trigger a search error message and increment the recent error volume.
     213     */
     214    protected function search_error( $reason ) {
     215        trigger_error( 'Plugin directory search: '.$reason, E_USER_WARNING );
     216        return $this->increment_error_volume();
     217    }
    169218
    170219    /*
     
    185234        // Other processes will use the stale cached value if it's present, even for a while after the expiration time if a fresh value is still being fetched.
    186235        if ( wp_cache_add( $lock_key, 1, self::CACHE_GROUP, 15 ) ) {
    187             $request = wp_remote_post( $service_url, array(
    188                 'headers' => array(
    189                     'Content-Type' => 'application/json',
    190                 ),
    191                 'timeout' => 10,
    192                 'user-agent' => 'jetpack_search',
    193                 'body' => $json_es_args,
    194             ) );
     236
     237            // If the error volume is high, there's a proportionally lower chance that we'll actually attempt to hit the API.
     238            if ( $this->error_volume_is_low() ) {
     239                $request = wp_remote_post( $service_url, array(
     240                    'headers' => array(
     241                        'Content-Type' => 'application/json',
     242                    ),
     243                    'timeout' => 10,
     244                    'user-agent' => 'jetpack_search',
     245                    'body' => $json_es_args,
     246                ) );
     247            } else {
     248                trigger_error( 'Plugin directory search: skipping search due to high error volume', E_USER_WARNING );
     249                // Hopefully we still have a cached response to return
     250                return $response;
     251            }
    195252
    196253            // If there's a network or HTTP error, we'll intentionally leave the temporary lock to expire in a few seconds.
     
    202259
    203260                if ( is_wp_error( $request ) )
    204                     trigger_error( 'Plugin directory search: http error '.$request->get_error_message(), E_USER_WARNING );
     261                    $this->search_error( 'http error '.$request->get_error_message(), E_USER_WARNING );
    205262                else
    206                     trigger_error( 'Plugin directory search: http status '.wp_remote_retrieve_response_code( $request ), E_USER_WARNING );
     263                    $this->search_error( 'http status '.wp_remote_retrieve_response_code( $request ), E_USER_WARNING );
    207264
    208265                // If we have a stale cached response, return that. Otherwise, return the error object.
     
    219276
    220277                if ( isset( $fresh_response['error'] ) )
    221                     trigger_error( 'Plugin directory search: remote error '.$fresh_response['error'], E_USER_WARNING );
     278                    $this->search_error( 'remote error '.$fresh_response['error'], E_USER_WARNING );
    222279                else
    223                     trigger_error( 'Plugin directory search: invalid json response', E_USER_WARNING );
     280                    $this->search_error( 'invalid json response', E_USER_WARNING );
    224281
    225282                // Return a stale response if we have one
Note: See TracChangeset for help on using the changeset viewer.