Changeset 6022

Timestamp:

10/13/2017 08:56:56 AM (9 years ago)

Author:

dd32

Message:

Plugin Directory: Generate md5 hashes for plugins.

This is a POC and may change or be removed in the future, it's here for testing purposes.

A api.wordpress.org endpoint may be available in the future to access it.
This is only enabled for the 'exploit-scanner' plugin at present, purely for testing, as it publishes the md5 hashes of its own files already

Compare ​https://wordpress.org/plugins/exploit-scanner/ to ​https://downloads.wordpress.org/plugins/exploit-scanner.1.5.2.checksums.json

See #3192

Location:

sites/trunk/wordpress.org/public_html/wp-content/plugins/plugin-directory

Files:

• bin/rebuild-zip.php (modified) (3 diffs)
• cli/class-import.php (modified) (1 diff)
• zip/class-builder.php (modified) (8 diffs)
• zip/class-serve.php (modified) (4 diffs)

sites/trunk/wordpress.org/public_html/wp-content/plugins/plugin-directory/bin/rebuild-zip.php

@@ -1 +1 @@
 <?php
 namespace WordPressdotorg\Plugin_Directory;
+use WordPressdotorg\Plugin_Directory\Plugin_Directory;
 use WordPressdotorg\Plugin_Directory\Tools\SVN;
 
@@ -86 +87 @@
         $zip_builder = new ZIP\Builder();
 
+        $plugin_post = Plugin_Directory::get_plugin_post( $plugin_slug );
+        if ( ! $plugin_post ) {
+                throw new Exception( "Could not locate plugin post" );
+        }
+        $stable_tag = get_post_meta( $plugin_post->ID, 'stable_tag', true ) ?? 'trunk';
+
         // (re)Build & Commit 5 Zips at a time to avoid limitations.
         foreach ( array_chunk( $versions, 5 ) as $versions_to_build ) {
@@ -91 +98 @@
                         $plugin_slug,
                         $versions_to_build,
-                        "{$plugin_slug}: Rebuild triggered by " . php_uname('n' )
+                        "{$plugin_slug}: Rebuild triggered by " . php_uname('n' ),
+                        $stable_tag
                 );
         }

sites/trunk/wordpress.org/public_html/wp-content/plugins/plugin-directory/cli/class-import.php

@@ -196 +196 @@
                                 $svn_revision_triggered ?
                                         "{$plugin_slug}: ZIP build triggered by https://plugins.trac.wordpress.org/changeset/{$svn_revision_triggered}" :
-                                        "{$plugin_slug}: ZIP build triggered by " . php_uname('n')
+                                        "{$plugin_slug}: ZIP build triggered by " . php_uname('n'),
+                                $stable_tag
                         );
                 } catch( Exception $e ) {

sites/trunk/wordpress.org/public_html/wp-content/plugins/plugin-directory/zip/class-builder.php

@@ -15 +15 @@
         const ZIP_SVN_URL = PLUGIN_ZIP_SVN_URL;
 
-        protected $zip_file = '';
-        protected $tmp_build_dir  = '';
-        protected $tmp_dir = '';
-
-        protected $slug    = '';
-        protected $version = '';
-        protected $context = '';
-
-        /**
-         * Generate a ZIP for a provided Plugin versions.
+        protected $zip_file      = '';
+        protected $checksum_file = '';
+        protected $tmp_build_dir = '';
+        protected $tmp_dir       = '';
+
+        protected $slug       = '';
+        protected $version    = '';
+        protected $context    = '';
+        protected $stable_tag = '';
+
+        /**
+         * Generate a ZIP for a provided Plugin tags.
          *
          * @param string $slug     The plugin slug.
@@ -30 +32 @@
          * @param string $context  The context of this Builder instance (commit #, etc)
          */
-        public function build( $slug, $versions, $context = '' ) {
+        public function build( $slug, $versions, $context = '', $stable_tag = '' ) {
                 // Bail when in an unconfigured environment.
                 if ( ! defined( 'PLUGIN_ZIP_SVN_URL' ) ) {
@@ -36 +38 @@
                 }
 
-                $this->slug     = $slug;
-                $this->versions = $versions;
-                $this->context  = $context;
+                $this->slug       = $slug;
+                $this->versions   = $versions;
+                $this->context    = $context;
+                $this->stable_tag = $stable_tag;
 
                 // General TMP directory
@@ -97 +100 @@
                         // Pull the ZIP file down we're going to modify, which may not already exist.
                         SVN::up( $this->zip_file );
+                        // This is done within the checksum generation function due to us not knowing the checksum filename until export_plugin().
+                        // SVN::up( $this->checksum_file );
 
                         try {
@@ -104 +109 @@
 
                                 $this->export_plugin();
-                                $this->fix_directory_dates();                   
+                                $this->fix_directory_dates();
+
                                 $this->generate_zip();
+
+                                $this->generate_checksums();
+
                                 $this->cleanup_plugin_tmp();
 
@@ -114 +123 @@
                                 // Perform an SVN up to revert any changes made.
                                 SVN::up( $this->zip_file );
+                                if ( $this->checksum_file ) {
+                                        SVN::up( $this->checksum_file );
+                                }
                                 continue;
                         }
 
                         // Add the ZIP file to SVN - This is only really needed for new files which don't exist in SVN.
-                        SVN::add( $this->zip_file );                    
+                        SVN::add( $this->zip_file );
+                        if ( $this->checksum_file ) {
+                                SVN::add( $this->checksum_file );
+                        }
                 }
 
@@ -146 +161 @@
 
         /**
+         * Generates a JSON file containing the checksums of the files within the ZIP.
+         *
+         * In the event that a previous ZIP for this version exists, checksums for all versions of the file will be included.
+         */
+        function generate_checksums() {
+                // Only enable this for the `exploit-scanner` plugin for the time being.
+                if ( 'exploit-scanner' != $this->slug ) {
+                        return;
+                }
+
+                // Don't create checksums for trunk.
+                if ( ! $this->stable_tag || ( 'trunk' == $this->version && 'trunk' != $this->stable_tag && '' != $this->stable_tag ) ) {
+                        return;
+                }
+
+                // Fetch the plugin headers
+                $plugin_data = false;
+                foreach ( glob( $this->tmp_build_dir . '/' . $this->slug . '/*.php' ) as $filename ) {
+                        $plugin_data = get_plugin_data( $filename, false, false );
+
+                        if ( $plugin_data['Name'] && '' !== $plugin_data['Version'] ) {
+                                break;
+                        }
+                }
+
+                if ( ! $plugin_data || '' === $plugin_data['Version'] ) {
+                        return;
+                }
+
+                $plugin_version = $plugin_data['Version'];
+                // Catch malformed version strings.
+                if ( basename( $plugin_version ) != $plugin_version ) {
+                        return;
+                }
+
+                $this->checksum_file = "{$this->tmp_dir}/{$this->slug}/{$this->slug}.{$plugin_version}.checksums.json";
+
+                // Checkout the Checksum file for this plugin version
+                SVN::up( $this->checksum_file );
+
+                // Existing checksums?
+                $existing_json_checksum_file = file_exists( $this->checksum_file );
+
+                $this->exec( sprintf(
+                        'cd %s && find . -type f -print0 | sort -z | xargs -0 md5sum 2>&1',
+                        escapeshellarg( $this->tmp_build_dir . '/' . $this->slug )
+                ), $checksum_output, $return_value );
+
+                if ( $return_value ) {
+                //      throw new Exception( __METHOD__ . ': Checksum generation failed, return code: ' . $return_value, 503 );
+                // For now, just silently bail.
+                        return;
+                }
+
+                $checksums = array();
+                foreach ( $checksum_output as $line ) {
+                        list( $md5, $filename ) = preg_split( '!\s+!', $line );
+                        $filename = preg_replace( '!^./!', '', $filename );
+                        $checksums[ trim( $filename ) ] = trim( $md5 );
+                }
+
+                $json_checksum_file = (object) array(
+                        'plugin'     => $this->slug,
+                        'version'    => $plugin_version,
+                        'source_tag' => $this->version,
+                        'zip'        => basename( $this->zip_file ),
+                        'checksums'  => $checksums
+                );
+
+                // If the checksum file exists already, merge it into this one.
+                if ( $existing_json_checksum_file ) {
+                        $existing_json_checksum_file = json_decode( file_get_contents( $this->checksum_file ) );
+
+                        if ( $existing_json_checksum_file && ! empty( $existing_json_checksum_file->checksums ) ) {
+                                foreach ( $existing_json_checksum_file->checksums as $file => $checksum_details ) {
+
+                                        if ( ! isset( $json_checksum_file->checksums[ $file ] ) ) {
+                                                // Deleted file, include it in checksums.
+                                                $json_checksum_file->checksums[ $file ] = $checksum_details;
+
+                                        } elseif ( $json_checksum_file->checksums[ $file ] != $checksum_details ) {
+                                                // Checksum has changed, include both in the resulting json file.
+                                                if ( is_array( $checksum_details ) ) {
+                                                        $checksum_details[] = $json_checksum_file->checksums[ $file ];
+                                                        $json_checksum_file->checksums[ $file ] = $checksum_details;
+                                                } else {
+                                                        $json_checksum_file->checksums[ $file ] = array(
+                                                                $json_checksum_file->checksums[ $file ],
+                                                                $checksum_details
+                                                        );
+                                                }
+                                        }
+                                }
+                        }
+                }
+
+                file_put_contents( $this->checksum_file, wp_json_encode( $json_checksum_file ) );
+        }
+
+        /**
          * Generates a temporary unique directory in a given directory
          *
@@ -279 +394 @@
          */
         public function invalidate_zip_caches( $versions ) {
-                // TODO: Implement PURGE 
+                // TODO: Implement PURGE
                 return true;
                 if ( ! defined( 'PLUGIN_ZIP_X_ACCEL_REDIRECT_LOCATION' ) ) {

sites/trunk/wordpress.org/public_html/wp-content/plugins/plugin-directory/zip/class-serve.php

@@ -37 +37 @@
                 $zip = basename( parse_url( $_SERVER['REQUEST_URI'], PHP_URL_PATH ) );
 
-                $slug = false;
+                if ( ! preg_match( "!^(?P<slug>[a-z0-9-_]+)(\.(?P<version>.+?))?\.(?P<request_type>zip|checksums\.json)$!i", $zip, $m ) ) {
+                        throw new Exception( __METHOD__ . ": Invalid URL." );
+                }
+
+                $slug = strtolower( $m['slug'] );
+
                 $version = 'trunk';
-
-                if ( ! preg_match( "!^(?P<slug>[a-z0-9-_]+)(.(?P<version>.+))?.zip$!i", $zip, $m ) ) {
-                        throw new Exception( __METHOD__ . ": Invalid URL." );
-                }
-
-                $slug = strtolower( $m['slug'] );
-                if ( isset( $m['version'] ) ) {
+                if ( isset( $m['version'] ) && '' !== $m['version'] ) {
                         $version = $m['version'];
                 }
-
                 if ( 'latest-stable' == $version ) {
                         $version = $this->get_stable_tag( $slug );
+                }
+
+                if ( 'zip' == strtolower( $m['request_type'] ) ) {
+                        $checksum_request = false;
+                } else {
+                        $checksum_request = true;
+
+                        // Checksum requests for 'trunk' are not possible.
+                        if ( 'trunk' == $version ) {
+                                throw new Exception( __METHOD__ . ": Checksum requests must include a version." );
+                        }
+
                 }
 
@@ -56 +66 @@
                         'stats' => true,
                 );
-                if ( isset( $_GET['stats'] ) ) {
+
+                if ( $checksum_request ) {
+                        $args['stats'] = false;
+
+                } elseif ( isset( $_GET['stats'] ) ) {
                         $args['stats'] = (bool) $_GET['stats'];
+
                 } elseif ( isset( $_GET['nostats'] ) ) {
                         $args['stats'] = !empty( $_GET['nostats'] );
                 }
 
-                return compact( 'zip', 'slug', 'version', 'args' );
+                return compact( 'zip', 'slug', 'version', 'args', 'checksum_request' );
         }
 
@@ -118 +133 @@
 
         /**
-         * Returns the files to use for the request.
+         * Returns the file to be served for the request.
          *
          * @param array $request The request object for the request.
-         * @return array An array containing the files to use for the request, 'zip' and 'md5'.
+         * @return array The file to serve.
          */
         protected function get_file( $request ) {
-                if ( empty( $request['version'] ) || 'trunk' == $request['version'] ) {
+                // Checksum requests must include a version
+                if ( $request['checksum_request'] ) {
+                        return "{$request['slug']}/{$request['slug']}.{$request['version']}.checksums.json";
+
+                } elseif ( empty( $request['version'] ) || 'trunk' == $request['version'] ) {
                         return "{$request['slug']}/{$request['slug']}.zip";
+
                 } else {
                         return "{$request['slug']}/{$request['slug']}.{$request['version']}.zip";
                 }
+
         }
 
@@ -137 +158 @@
          */
         protected function serve_zip( $request ) {
-                $zip = $this->get_file( $request );
+                $file = $this->get_file( $request );
 
                 if ( defined( 'PLUGIN_ZIP_X_ACCEL_REDIRECT_LOCATION' ) ) {
-                        $zip_url = PLUGIN_ZIP_X_ACCEL_REDIRECT_LOCATION . $zip;
-
-                        header( 'Content-Type: application/zip' );
-                        header( 'Content-Disposition: attachment; filename=' . basename( $zip ) );
-                        header( "X-Accel-Redirect: $zip_url" );
+                        $file_url = PLUGIN_ZIP_X_ACCEL_REDIRECT_LOCATION . $file;
+
+                        if ( $request['checksum_request'] ) {
+                                header( 'Content-Type: application/json' );
+                        } else {
+                                header( 'Content-Type: application/zip' );
+                                header( 'Content-Disposition: attachment; filename=' . basename( $file ) );
+                        }
+                        header( "X-Accel-Redirect: $file_url" );
                 } else {
                         header( 'Content-Type: text/plain' );
-                        echo "This is a request for $zip, this server isn't currently configured to serve zip files.\n";
+                        echo "This is a request for $file, this server isn't currently configured to serve files.\n";
                 }