Index: wp-content/plugins/wporg-markdown/README.md =================================================================== --- wp-content/plugins/wporg-markdown/README.md (nonexistent) +++ wp-content/plugins/wporg-markdown/README.md (working copy) @@ -0,0 +1,47 @@ +# WPORG Markdown Importer + +Imports Markdown from a remote site (like GitHub) into WordPress as pages. + +## Configuration + +Each importer needs to override the abstract methods: + +* `get_base()` - Base URL for imported pages. This will be stripped from the key before comparing. +* `get_manifest_url()` - URL pointing to the manifest. +* `get_post_type()` - Post type to import as. + +## Manifest Format + +The manifest should be a JSON object, with the keys set to the desired permalink (excluding the base path). Each item should also be a JSON object, containing the following keys: + +* `slug` - Post name to insert. (Must match the final path-part of the key.) +* `markdown_source` - URL for the Markdown file to parse into content. +* `parent` - Key for the parent to store under. (Must correspond to the non-final path-parts of the key.) +* `title` - Title to use when creating post. Used temporarily, will be updated from the Markdown file. If not specified, defaults to `slug` (but will be updated from Markdown source). + +**Note:** The Handbook index should have the slug `index`. + +Example: + +```json +{ + "foo": { + "title": "Temporary Foo Title", + "slug": "foo", + "markdown_source": "https://raw.githubusercontent.com/WordPress/doc-repo/master/foo.md", + "parent": null + }, + "foo/bar": { + "title": "Temporary Bar Title", + "slug": "bar", + "markdown_source": "https://raw.githubusercontent.com/WordPress/doc-repo/master/foo/bar.md", + "parent": "foo" + }, + "foo/bar/quux": { + "title": "Temporary Quux Title", + "slug": "quux", + "markdown_source": "https://raw.githubusercontent.com/WordPress/doc-repo/master/foo/bar/quux.md", + "parent": "foo/bar" + } +} +``` Index: wp-content/plugins/wporg-markdown/inc/class-importer.php =================================================================== --- wp-content/plugins/wporg-markdown/inc/class-importer.php (nonexistent) +++ wp-content/plugins/wporg-markdown/inc/class-importer.php (working copy) @@ -0,0 +1,332 @@ +get_base(), '', get_permalink( $post->ID ) ), '/' ); + if ( empty( $key ) ) { + $key = 'index'; + } + + $data = array( + 'post_id' => $post->ID, + ); + return array( $key, $data ); + } + + /** + * Import the manifest. + * + * Fetches the manifest, parses, and creates pages as needed. + */ + public function import_manifest() { + $response = wp_remote_get( $this->get_manifest_url() ); + if ( is_wp_error( $response ) ) { + if ( class_exists( 'WP_CLI' ) ) { + WP_CLI::error( $response->get_error_message() ); + } + return $response; + } elseif ( 200 !== wp_remote_retrieve_response_code( $response ) ) { + if ( class_exists( 'WP_CLI' ) ) { + WP_CLI::error( 'Non-200 from Markdown source' ); + } + return new WP_Error( 'invalid-http-code', 'Markdown source returned non-200 http code.' ); + } + $manifest = json_decode( wp_remote_retrieve_body( $response ), true ); + if ( ! $manifest ) { + if ( class_exists( 'WP_CLI' ) ) { + WP_CLI::error( 'Invalid manifest' ); + } + return new WP_Error( 'invalid-manifest', 'Manifest did not unfurl properly.' );; + } + // Fetch all handbook posts for comparison + $q = new WP_Query( array( + 'post_type' => $this->get_post_type(), + 'post_status' => 'publish', + 'posts_per_page' => $this->posts_per_page, + ) ); + $existing = array(); + foreach ( $q->posts as $post ) { + list( $key, $data ) = $this->get_existing_for_post( $post ); + $existing[ $key ] = $data; + } + $created = $updated = 0; + foreach ( $manifest as $key => $doc ) { + // Already exists, update. + if ( ! empty( $existing[ $key ] ) ) { + $existing_id = $existing[ $key ]['post_id']; + if ( $this->update_post_from_manifest_doc( $existing_id, $doc ) ) { + $updated++; + } + + continue; + } + if ( $this->process_manifest_doc( $doc, $existing, $manifest ) ) { + $created++; + } + } + if ( class_exists( 'WP_CLI' ) ) { + WP_CLI::success( "Successfully created {$created} and updated {$updated} handbook pages." ); + } + } + + /** + * Process a document from the manifest. + * + * @param array $doc Document to process. + * @param array $existing List of existing posts, will be added to. + * @param array $manifest Manifest data. + * @return boolean True if processing succeeded, false otherwise. + */ + protected function process_manifest_doc( $doc, &$existing, $manifest ) { + $post_parent = null; + if ( ! empty( $doc['parent'] ) ) { + // Find the parent in the existing set + if ( empty( $existing[ $doc['parent'] ] ) ) { + if ( ! $this->process_manifest_doc( $manifest[ $doc['parent'] ], $existing, $manifest ) ) { + return false; + } + } + if ( ! empty( $existing[ $doc['parent'] ] ) ) { + $parent = $existing[ $doc['parent'] ]; + $post_parent = $parent['post_id']; + } + } + $post = $this->create_post_from_manifest_doc( $doc, $post_parent ); + if ( $post ) { + list( $key, $data ) = $this->get_existing_for_post( $post ); + $existing[ $key ] = $data; + return true; + } + return false; + } + + /** + * Create a new handbook page from the manifest document + */ + protected function create_post_from_manifest_doc( $doc, $post_parent = null ) { + if ( $doc['slug'] === 'index' ) { + $doc['slug'] = $this->get_post_type(); + } + $post_data = array( + 'post_type' => $this->get_post_type(), + 'post_status' => 'publish', + 'post_parent' => $post_parent, + 'post_title' => wp_slash( $doc['slug'] ), + 'post_name' => sanitize_title_with_dashes( $doc['slug'] ), + ); + if ( isset( $doc['title'] ) ) { + $doc['post_title'] = sanitize_text_field( wp_slash( $doc['title'] ) ); + } + $post_id = wp_insert_post( $post_data ); + if ( ! $post_id ) { + return false; + } + if ( class_exists( 'WP_CLI' ) ) { + WP_CLI::log( "Created post {$post_id} for {$doc['slug']}." ); + } + update_post_meta( $post_id, $this->meta_key, esc_url_raw( $doc['markdown_source'] ) ); + return get_post( $post_id ); + } + + /** + * Update an existing post from the manifest. + * + * @param int $post_id Existing post ID. + * @param array $doc Document details from the manifest. + * @return boolean True if updated, false otherwise. + */ + protected function update_post_from_manifest_doc( $post_id, $doc ) { + $did_update = update_post_meta( $post_id, $this->meta_key, esc_url_raw( $doc['markdown_source'] ) ); + return $did_update; + } + + /** + * Update existing posts from Markdown source. + * + * Reparses the Markdown for every page. + */ + public function import_all_markdown() { + $q = new WP_Query( array( + 'post_type' => $this->get_post_type(), + 'post_status' => 'publish', + 'fields' => 'ids', + 'posts_per_page' => $this->posts_per_page, + ) ); + $ids = $q->posts; + $success = 0; + foreach( $ids as $id ) { + $ret = $this->update_post_from_markdown_source( $id ); + if ( class_exists( 'WP_CLI' ) ) { + if ( is_wp_error( $ret ) ) { + WP_CLI::warning( $ret->get_error_message() ); + } elseif ( false === $ret ) { + WP_CLI::log( "No updates for {$id}" ); + $success++; + } else { + WP_CLI::log( "Updated {$id} from markdown source" ); + $success++; + } + } + } + if ( class_exists( 'WP_CLI' ) ) { + $total = count( $ids ); + WP_CLI::success( "Successfully updated {$success} of {$total} pages." ); + } + } + + /** + * Update a post from its Markdown source. + * + * @param int $post_id Post ID to update. + * @return boolean|WP_Error True if updated, false if no update needed, error otherwise. + */ + protected function update_post_from_markdown_source( $post_id ) { + $markdown_source = $this->get_markdown_source( $post_id ); + if ( is_wp_error( $markdown_source ) ) { + return $markdown_source; + } + if ( ! function_exists( 'jetpack_require_lib' ) ) { + return new WP_Error( 'missing-jetpack-require-lib', 'jetpack_require_lib() is missing on system.' ); + } + + // Transform GitHub repo HTML pages into their raw equivalents + $markdown_source = preg_replace( '#https?://github\.com/([^/]+/[^/]+)/blob/(.+)#', 'https://raw.githubusercontent.com/$1/$2', $markdown_source ); + $markdown_source = add_query_arg( 'v', time(), $markdown_source ); + + // Grab the stored ETag, and use it to deduplicate. + $args = array( + 'headers' => array(), + ); + $last_etag = get_post_meta( $post_id, $this->etag_meta_key, true ); + if ( ! empty( $last_etag ) ) { + $args['headers']['If-None-Match'] = $last_etag; + } + + $response = wp_remote_get( $markdown_source, $args ); + if ( is_wp_error( $response ) ) { + return $response; + } elseif ( 304 === wp_remote_retrieve_response_code( $response ) ) { + // No update required! + return false; + } elseif ( 200 !== wp_remote_retrieve_response_code( $response ) ) { + return new WP_Error( 'invalid-http-code', 'Markdown source returned non-200 http code.' ); + } + + $etag = wp_remote_retrieve_header( $response, 'etag' ); + + $markdown = wp_remote_retrieve_body( $response ); + // Strip YAML doc from the header + $markdown = preg_replace( '#^---(.+)---#Us', '', $markdown ); + + $title = null; + if ( preg_match( '/^#\s(.+)/', $markdown, $matches ) ) { + $title = $matches[1]; + $markdown = preg_replace( '/^#\swp\s(.+)/', '', $markdown ); + } + $markdown = trim( $markdown ); + + // Steal the first sentence as the excerpt + $excerpt = ''; + if ( preg_match( '/^(.+)/', $markdown, $matches ) ) { + $excerpt = $matches[1]; + $markdown = preg_replace( '/^(.+)/', '', $markdown ); + } + + // Transform to HTML and save the post + jetpack_require_lib( 'markdown' ); + $parser = new WPCom_GHF_Markdown_Parser(); + $html = $parser->transform( $markdown ); + $post_data = array( + 'ID' => $post_id, + 'post_content' => wp_filter_post_kses( wp_slash( $html ) ), + 'post_excerpt' => sanitize_text_field( wp_slash( $excerpt ) ), + ); + if ( ! is_null( $title ) ) { + $post_data['post_title'] = sanitize_text_field( wp_slash( $title ) ); + } + wp_update_post( $post_data ); + + // Set ETag for future updates. + update_post_meta( $post_id, $this->etag_meta_key, wp_slash( $etag ) ); + + return true; + } + + /** + * Retrieve the markdown source URL for a given post. + */ + public function get_markdown_source( $post_id ) { + $markdown_source = get_post_meta( $post_id, $this->meta_key, true ); + if ( ! $markdown_source ) { + return new WP_Error( 'missing-markdown-source', 'Markdown source is missing for post.' ); + } + + return $markdown_source; + } +} Index: wp-content/plugins/wporg-markdown/plugin.php =================================================================== --- wp-content/plugins/wporg-markdown/plugin.php (nonexistent) +++ wp-content/plugins/wporg-markdown/plugin.php (working copy) @@ -0,0 +1,8 @@ +set( 'posts_per_page', 250 ); } } - - public static function action_devhub_cli_manifest_import() { - $response = wp_remote_get( self::$commands_manifest ); - if ( is_wp_error( $response ) ) { - return $response; - } elseif ( 200 !== wp_remote_retrieve_response_code( $response ) ) { - return new WP_Error( 'invalid-http-code', 'Markdown source returned non-200 http code.' ); - } - $manifest = json_decode( wp_remote_retrieve_body( $response ), true ); - if ( ! $manifest ) { - return new WP_Error( 'invalid-manifest', 'Manifest did not unfurl properly.' );; - } - // Fetch all handbook posts for comparison - $q = new WP_Query( array( - 'post_type' => self::$supported_post_types, - 'post_status' => 'publish', - 'posts_per_page' => self::$posts_per_page, - ) ); - $existing = array(); - foreach( $q->posts as $post ) { - $cmd_path = rtrim( str_replace( home_url( 'cli/commands/' ), '', get_permalink( $post->ID ) ), '/' ); - $existing[ $cmd_path ] = array( - 'post_id' => $post->ID, - 'cmd_path' => $cmd_path, - ); - } - $created = 0; - foreach( $manifest as $doc ) { - // Already exists - if ( wp_filter_object_list( $existing, array( 'cmd_path' => $doc['cmd_path'] ) ) ) { - continue; - } - if ( self::process_manifest_doc( $doc, $existing, $manifest ) ) { - $created++; - } - } - if ( class_exists( 'WP_CLI' ) ) { - \WP_CLI::success( "Successfully created {$created} handbook pages." ); - } - } - - private static function process_manifest_doc( $doc, &$existing, $manifest ) { - $post_parent = null; - if ( ! empty( $doc['parent'] ) ) { - // Find the parent in the existing set - $parents = wp_filter_object_list( $existing, array( 'cmd_path' => $doc['parent'] ) ); - if ( empty( $parents ) ) { - if ( ! self::process_manifest_doc( $manifest[ $doc['parent'] ], $existing, $manifest ) ) { - return; - } - $parents = wp_filter_object_list( $existing, array( 'cmd_path' => $doc['parent'] ) ); - } - if ( ! empty( $parents ) ) { - $parent = array_shift( $parents ); - $post_parent = $parent['post_id']; - } - } - $post = self::create_post_from_manifest_doc( $doc, $post_parent ); - if ( $post ) { - $cmd_path = rtrim( str_replace( home_url( 'cli/commands/' ), '', get_permalink( $post->ID ) ), '/' ); - $existing[ $cmd_path ] = array( - 'post_id' => $post->ID, - 'cmd_path' => $cmd_path, - ); - return true; - } - return false; - } - - public static function action_devhub_cli_markdown_import() { - $q = new WP_Query( array( - 'post_type' => self::$supported_post_types, - 'post_status' => 'publish', - 'fields' => 'ids', - 'posts_per_page' => self::$posts_per_page, - ) ); - $ids = $q->posts; - $success = 0; - foreach( $ids as $id ) { - $ret = self::update_post_from_markdown_source( $id ); - if ( class_exists( 'WP_CLI' ) ) { - if ( is_wp_error( $ret ) ) { - \WP_CLI::warning( $ret->get_error_message() ); - } else { - \WP_CLI::log( "Updated {$id} from markdown source" ); - $success++; - } - } - } - if ( class_exists( 'WP_CLI' ) ) { - $total = count( $ids ); - \WP_CLI::success( "Successfully updated {$success} of {$total} CLI command pages." ); - } - } - - /** - * Create a new handbook page from the manifest document - */ - private static function create_post_from_manifest_doc( $doc, $post_parent = null ) { - $post_data = array( - 'post_type' => 'command', - 'post_status' => 'publish', - 'post_parent' => $post_parent, - 'post_title' => sanitize_text_field( wp_slash( $doc['title'] ) ), - 'post_name' => sanitize_title_with_dashes( $doc['slug'] ), - ); - $post_id = wp_insert_post( $post_data ); - if ( ! $post_id ) { - return false; - } - if ( class_exists( 'WP_CLI' ) ) { - \WP_CLI::log( "Created post {$post_id} for {$doc['title']}." ); - } - update_post_meta( $post_id, self::$meta_key, esc_url_raw( $doc['markdown_source'] ) ); - return get_post( $post_id ); - } - - /** - * Update a post from its Markdown source - */ - private static function update_post_from_markdown_source( $post_id ) { - $markdown_source = self::get_markdown_source( $post_id ); - if ( is_wp_error( $markdown_source ) ) { - return $markdown_source; - } - if ( ! function_exists( 'jetpack_require_lib' ) ) { - return new WP_Error( 'missing-jetpack-require-lib', 'jetpack_require_lib() is missing on system.' ); - } - - // Transform GitHub repo HTML pages into their raw equivalents - $markdown_source = preg_replace( '#https?://github\.com/([^/]+/[^/]+)/blob/(.+)#', 'https://raw.githubusercontent.com/$1/$2', $markdown_source ); - $markdown_source = add_query_arg( 'v', time(), $markdown_source ); - $response = wp_remote_get( $markdown_source ); - if ( is_wp_error( $response ) ) { - return $response; - } elseif ( 200 !== wp_remote_retrieve_response_code( $response ) ) { - return new WP_Error( 'invalid-http-code', 'Markdown source returned non-200 http code.' ); - } - - $markdown = wp_remote_retrieve_body( $response ); - // Strip YAML doc from the header - $markdown = preg_replace( '#^---(.+)---#Us', '', $markdown ); - - $title = null; - if ( preg_match( '/^#\s(.+)/', $markdown, $matches ) ) { - $title = $matches[1]; - $markdown = preg_replace( '/^#\swp\s(.+)/', '', $markdown ); - } - $markdown = trim( $markdown ); - - // Steal the first sentence as the excerpt - $excerpt = ''; - if ( preg_match( '/^(.+)/', $markdown, $matches ) ) { - $excerpt = $matches[1]; - $markdown = preg_replace( '/^(.+)/', '', $markdown ); - } - - // Transform to HTML and save the post - jetpack_require_lib( 'markdown' ); - $parser = new \WPCom_GHF_Markdown_Parser; - $html = $parser->transform( $markdown ); - $post_data = array( - 'ID' => $post_id, - 'post_content' => wp_filter_post_kses( wp_slash( $html ) ), - 'post_excerpt' => sanitize_text_field( wp_slash( $excerpt ) ), - ); - if ( ! is_null( $title ) ) { - $post_data['post_title'] = sanitize_text_field( wp_slash( $title ) ); - } - wp_update_post( $post_data ); - return true; - } - - /** - * Retrieve the markdown source URL for a given post. - */ - public static function get_markdown_source( $post_id ) { - $markdown_source = get_post_meta( $post_id, self::$meta_key, true ); - if ( ! $markdown_source ) { - return new WP_Error( 'missing-markdown-source', 'Markdown source is missing for post.' ); - } - - return $markdown_source; - } - } -DevHub_CLI::init(); - +DevHub_CLI::instance()->init(); Index: wp-content/themes/pub/wporg-developer/inc/rest-api.php =================================================================== --- wp-content/themes/pub/wporg-developer/inc/rest-api.php (nonexistent) +++ wp-content/themes/pub/wporg-developer/inc/rest-api.php (working copy) @@ -0,0 +1,55 @@ +init(); +