diff --git a/CLAUDE.md b/CLAUDE.md index 0e8ecb3..35bf057 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -18,18 +18,94 @@ This is a WordPress plugin that embeds Fourthwall store products into WordPress ### Key Components -**HTTP Request Layer** (`fwembed_make_request` in shortcode.php): +**HTTP Request Layer** (`fwembed_make_request` / `fwembed_make_requests` in shortcode.php): - Centralized cURL-based HTTP client with browser-like headers to avoid 403 blocks -- 1-hour transient caching using WordPress transients (cache key: `fwembed_{md5(url)}`) -- SSL verification configurable via admin settings -- Cookie handling via `/tmp/cookies.txt` for session persistence +- Stale-while-revalidate transient caching (cache key: `fwembed_{md5(url)}`), with + a configurable freshness window - see **Caching Strategy** below +- `fwembed_make_requests()` fetches uncached URLs concurrently via `curl_multi` +- TLS certificate verification is always on and not configurable +- No cookie jar (see **Caching Strategy**) **HTML Parsing** (shortcode.php): - Uses DOMDocument/DOMXPath to extract product data from Fourthwall HTML -- Looks for `div[data-testid="product"]` elements +- Looks for `[data-testid="product"]` elements, falling back to `.product-tile` - Extracts product tiles, images, descriptions, titles, and prices via CSS class selectors - `loadHTML5()` wrapper ensures proper HTML5 parsing +**Single-product extraction is layered** (`fwembed_parse_single_product`) + +Each field is resolved independently from the most stable source available, so a +markup change can only degrade the field it actually touched: + +1. **JSON-LD** (`fwembed_product_from_jsonld`) - product pages embed a + `schema.org/Product` block. This is a published contract rather than styling, + so it survives theme changes. Source of truth for title, image, description + text, `sku`, `priceCurrency` and `availability`. +2. **Scraped markup** (`fwembed_product_from_html`) - fills gaps, and *wins* for + two fields on purpose: the price (it carries the store's own formatting, e.g. + "from $20") and the description (JSON-LD flattens it to plain text). +3. **OpenGraph `` tags** - last-resort title/image. + +`fwembed_find_jsonld_product` handles the shapes publishers emit: bare object, +top-level list, and `@graph` / `itemListElement` / `mainEntity` wrappers. +`fwembed_jsonld_offer` takes the cheapest offer, since variants each get their +own `Offer` and `AggregateOffer` states it as `lowPrice`. + +**Product discovery** (`fwembed_collect_product_urls`) + +`sitemap.xml` is the full catalog; the store page only renders whichever +collection it features. Sitemap first, page scrape as fallback - `[fourthwall_random +source="auto|sitemap|page"]` overrides. `fwembed_read_sitemap` follows one level +of sitemap index, caps at 10 nested sitemaps, and rejects any `` whose host +differs from the configured store so a hostile sitemap cannot redirect fetches. + +`robots.txt` allows `/products/*` and `/collections/all` for `*`; `/cart.js`, +`/checkout/*` and `/admin` are disallowed - do not fetch those. + +**Surviving Fourthwall markup changes** + +Fourthwall changes their theme markup without notice, and every past breakage has +been an over-specific selector. Three helpers exist to keep that from recurring — +use them for any new extraction: + +- `fwembed_class_predicate($class)` - builds a padded `contains()` predicate. + Never write `@class="foo"`; class order and extra classes change between releases. + (The `[fourthwall_single]` breakage was `//h2[@class="product-info__title"]` after + the title tag became `

` — match the class or `data-testid`, not the tag name.) +- `fwembed_query_first($xpath, [$q1, $q2, ...], $context)` - returns the first + matching query. Order selectors most-stable first: `data-testid` attributes, + then class names, then OpenGraph `` tags as a last resort. +- `fwembed_resolve_url($href, $page_url)` - resolves relative hrefs. Product links + are root-relative (`/products/foo`), so never concatenate onto the store URL: + that breaks on a trailing slash and on store URLs with a path (`/collections/all`). + +Known-stable hooks as of the last verification: the JSON-LD `Product` block, the +OpenGraph meta tags, and `data-testid` values `product`, `product.name`, +`product.price`, `product.image`. + +### Markup emitted for single products + +Tiles carry `data-availability` (`InStock` / `OutOfStock`) and `data-sku` from +JSON-LD, and gain a `product-tile--sold-out` class when out of stock. The class +is deliberately unstyled - it is a hook for themes, not a built-in badge. + +### Verifying against a live store + +There is no test suite in-repo. To check parsing after a Fourthwall change, load +`libs/shortcode.php` under PHP CLI with stubs for `get_option`, `get_transient`, +`set_transient`, `delete_transient`, `add_shortcode`, `add_action`, +`shortcode_atts`, `wp_next_scheduled`, `wp_schedule_single_event`, `esc_url`, +`esc_attr`, `esc_html`, `wpautop`, plus the `MINUTE_IN_SECONDS` / `DAY_IN_SECONDS` +constants. Then assert each shortcode returns a non-empty title, price, image and +a well-formed absolute link. Worth covering: + +- Store URL with and without a trailing slash, and as `/collections/all`. +- Degradation: strip the JSON-LD, then rename the CSS classes, then remove the + `og:` tags - the first two must still render, the last must render nothing. +- Freshness: entries are aged by rewriting the stored `expires_at`, **not** by + faking the clock - the plugin calls real `time()`, so a fake clock silently + tests nothing. + **Shortcodes**: - `[fourthwall]` - Displays all store products - `[fourthwall_single url="..." show_description="true"]` - Single product display @@ -45,7 +121,16 @@ This is a WordPress plugin that embeds Fourthwall store products into WordPress WordPress options API key: `fourthwall_settings_name` - `fourth_url` - Default Fourthwall store URL -- `ssl_verify` - Boolean for SSL certificate verification +- `cache_ttl` - Freshness window in **minutes** (default 60); `fwembed_cache_ttl()` + converts to seconds and falls back to 60 minutes for values below 1 + +Sanitised on save by `fourthwall_settings::sanitize_settings()`. + +The old `ssl_verify` option was removed - it was a development affordance, and +its checkbox never worked anyway (unchecked meant "key absent", which the read +treated as `true`). `fwembed_curl_handle()` now pins `CURLOPT_SSL_VERIFYPEER` +and `CURLOPT_SSL_VERIFYHOST`. Do not make certificate verification configurable +again; the plugin only ever talks to public HTTPS storefronts. ## Development Commands @@ -101,16 +186,51 @@ fourthwall-store-embed.zip - Set `$dom->documentURI` for proper relative URL resolution ### Caching Strategy -- All HTTP requests are cached for 1 hour (3600 seconds) -- Cache is stored in WordPress transients, not direct database access -- Error responses (non-200 status) are NOT cached -- Cache keys use `md5($url)` to handle special characters + +Stale-while-revalidate. A cache entry carries its own `expires_at`, and the +transient's own expiry is the much longer *retention* window: + +- **Fresh** (`now < expires_at`) - served directly. +- **Stale** (past `expires_at`, still stored) - served **immediately**, and a + WP-Cron job is queued to refresh it. Expiry therefore never lands the cost of + a network round trip on a visitor. +- **Absent** (past retention) - fetched synchronously. This is the only path + that blocks a page render. + +Consequences to keep in mind when changing this: + +- Never shorten transient expiry to the TTL. `fwembed_cache_retention()` + (>= 24h) is what keeps stale content available to serve; TTL only controls + when a refresh is triggered. +- `fwembed_schedule_refresh()` dedupes via `wp_next_scheduled()`, so a thousand + visitors hitting one stale entry queue a single job. `fwembed_do_refresh()` + additionally takes a `_lock` transient so two overlapping cron runs cannot + both fetch. +- Entries cached by older versions have no `expires_at`. They are read as stale, + served once, then rewritten in the new format - do not "clean up" that branch. +- Error responses (non-200) are still never cached. +- If `DISABLE_WP_CRON` is set with no real cron running, refreshes never fire and + content is served stale until retention lapses, then refetched synchronously. + +`fwembed_make_requests()` fetches everything uncached in one `curl_multi` batch +(8 concurrent, chunked). `[fourthwall_random]` uses it so a cold cache costs one +round trip instead of N sequential ones. Prefer it over looping +`fwembed_make_request()` whenever the URL set is known up front. + +**No cookie jar.** The old code shared `/tmp/cookies.txt` across every request +and every site on the host. It broke `curl_multi` (concurrent writes to one jar) +and was unnecessary - the storefront, product pages and sitemap all return 200 +without cookies. Do not reintroduce a shared jar; give each handle its own file +if session state ever becomes genuinely necessary. + +Cache keys use `md5($url)`; locks are `_lock`, so the admin "Clear Cache" +`LIKE '_transient_fwembed_%'` sweep clears both. ### Security Considerations - All user input sanitized via `esc_attr()`, `esc_html()`, `htmlspecialchars()` - Nonce verification for cache clearing: `wp_verify_nonce()` - Capability checks: `current_user_can('manage_options')` -- SSL verification enabled by default (disable only for local dev) +- TLS certificate verification is pinned on and cannot be disabled ### Version Management - Main plugin file contains placeholder: `Version: {auto_update_value_on_deploy}` diff --git a/README.md b/README.md index f86ee8c..ab60a70 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,14 @@ Displays a random selection of products from your store. - `count` (optional): Number of products to display (default: 3) - `urls` (optional): Comma-separated list of specific product URLs to randomize from - `store_url` (optional): Custom store URL (uses default from settings if not provided) +- `source` (optional): Where to discover products - `auto` (default), `sitemap`, or `page` + +**About `source`:** products are discovered from your store's `sitemap.xml`, which +lists your entire catalog. The older behaviour scraped the store page, which only +sees the collection that page happens to feature - so if your homepage shows 3 +featured items, `count="10"` could never return more than 3. `auto` uses the +sitemap and falls back to scraping the page if the sitemap is unavailable; use +`page` to force the old behaviour. #### Examples: @@ -58,11 +66,16 @@ Displays a random selection of products from your store. In the WordPress admin, go to Settings > Fourthwall Store Embed to set your Fourthwall store URL. +**Cache Lifetime** (default: 60 minutes) controls how often store content is +refreshed. Content is cached and served instantly; once it passes the lifetime it +is *still* served immediately while a background refresh runs, so raising or +lowering this never slows a page down. Use **Clear Cache** to force an immediate +refresh after changing products in Fourthwall. + ## Features - Caches requests for better performance - Responsive design -- SSL verification options - Error handling for failed requests - Random product selection - Support for multiple store URLs @@ -77,12 +90,10 @@ In the WordPress admin, go to Settings > Fourthwall Store Embed to set your Four **Store URL**: Enter your Fourthwall store URL (e.g., `https://your-store.fourthwall.com`) -**SSL Verification**: -- **Enabled (Recommended)**: Use for production sites to ensure secure connections -- **Disabled**: Use only for local development when SSL certificates are not properly configured +**Cache Lifetime**: How long store content stays fresh, in minutes (default: 60) **Cache Management**: -- Content is automatically cached for 1 hour to improve performance +- Content is cached and served instantly, then refreshed in the background - Use the "Clear Cache" button if products are not updating #### Display your entire store @@ -98,8 +109,8 @@ You can also display the product description by setting the `show_description` a ### Features -- **Smart Caching**: Automatic caching system reduces server load and improves performance -- **Configurable SSL**: Toggle SSL verification for development vs production environments +- **Smart Caching**: Stale content is served instantly while it refreshes in the background, so cache expiry never slows a page down +- **Parallel Fetching**: Uncached products are fetched concurrently rather than one at a time - **Error Handling**: Graceful fallbacks and clear error messages - **Admin Interface**: User-friendly settings page with clear instructions - **Cache Management**: Manual cache clearing for troubleshooting @@ -113,21 +124,21 @@ You can also display the product description by setting the `show_description` a ### Performance Notes -- Content is cached for 1 hour to reduce API calls to Fourthwall -- Cache automatically refreshes when content changes +- Content is cached to reduce requests to Fourthwall (default: 60 minutes) +- Once the cache lifetime passes, content is still served instantly while a + background refresh runs, so visitors never wait on the network - Manual cache clearing available in admin settings -- SSL verification can be disabled for local development ### Troubleshooting **Products not updating?** - Clear the cache using the "Clear Cache" button in admin settings - Check your store URL is correct -- Verify SSL verification setting matches your environment +- Lower the Cache Lifetime if you need changes to appear sooner -**SSL errors in development?** -- Disable SSL verification in admin settings (development only) -- Ensure proper SSL certificates in production +**Background refreshes not happening?** +- Refreshes run via WP-Cron. If `DISABLE_WP_CRON` is set with no system cron + configured, content is served stale until it is refetched on demand. **403 Forbidden errors?** - Fourthwall may be blocking automated requests diff --git a/libs/settings.php b/libs/settings.php index 1df17f3..774ee5e 100644 --- a/libs/settings.php +++ b/libs/settings.php @@ -27,7 +27,8 @@ class fourthwall_settings { register_setting( 'fourthwall_settings_group', - 'fourthwall_settings_name' + 'fourthwall_settings_name', + array( 'sanitize_callback' => array( $this, 'sanitize_settings' ) ) ); add_settings_section( @@ -46,15 +47,37 @@ class fourthwall_settings { ); add_settings_field( - 'ssl_verify', - __( 'SSL Verification', 'fourthwall_text_domain' ), - array( $this, 'render_ssl_verify_field' ), + 'cache_ttl', + __( 'Cache Lifetime', 'fourthwall_text_domain' ), + array( $this, 'render_cache_ttl_field' ), 'fourthwall_settings_name', 'fourthwall_settings_name_section' ); } + /** + * Sanitize settings before they are stored. + * + * @param array $input Raw submitted values + * @return array Cleaned values + */ + public function sanitize_settings( $input ) { + + $input = is_array( $input ) ? $input : array(); + $output = array(); + + if ( isset( $input['fourth_url'] ) ) { + $output['fourth_url'] = esc_url_raw( trim( $input['fourth_url'] ) ); + } + + $ttl = isset( $input['cache_ttl'] ) ? intval( $input['cache_ttl'] ) : 60; + $output['cache_ttl'] = $ttl < 1 ? 60 : $ttl; + + return $output; + + } + public function fourthwall_page_layout() { // Check required user capability @@ -94,7 +117,9 @@ class fourthwall_settings { echo '

' . __( 'Display random products:', 'fourthwall_text_domain' ) . ' [fourthwall_random count="5"]

' . "\n"; echo '

' . __( 'Random from specific URLs:', 'fourthwall_text_domain' ) . ' [fourthwall_random count="3" urls="https://store.com/product1,https://store.com/product2,https://store.com/product3"]

' . "\n"; echo '

' . __( 'Random from different store:', 'fourthwall_text_domain' ) . ' [fourthwall_random count="2" store_url="https://different-store.fourthwall.com"]

' . "\n"; - echo '

' . __( 'Note: Disable SSL verification only for local development. Keep enabled for production sites.', 'fourthwall_text_domain' ) . '

' . "\n"; + echo '

' . __( 'Random from the store page instead of the sitemap:', 'fourthwall_text_domain' ) . ' [fourthwall_random count="3" source="page"]

' . "\n"; + echo '

' . __( 'Random products are drawn from your store sitemap, which covers your whole catalog. Set source="page" to only use products shown on the store page itself.', 'fourthwall_text_domain' ) . '

' . "\n"; + echo '

' . __( 'Tip: point the Store URL at your /collections/all page to list every product with [fourthwall].', 'fourthwall_text_domain' ) . '

' . "\n"; echo ' ' . "\n"; echo '' . "\n"; @@ -114,17 +139,20 @@ class fourthwall_settings { } - function render_ssl_verify_field() { + function render_cache_ttl_field() { // Retrieve data from the database. $options = get_option( 'fourthwall_settings_name' ); // Set default value. - $value = isset( $options['ssl_verify'] ) ? $options['ssl_verify'] : ''; + $value = isset( $options['cache_ttl'] ) ? intval( $options['cache_ttl'] ) : 60; + if ( $value < 1 ) { + $value = 60; + } // Field output. - echo ''; - echo '

' . __( 'Enable SSL verification', 'fourthwall_text_domain' ) . '

'; + echo ' ' . esc_html__( 'minutes', 'fourthwall_text_domain' ); + echo '

' . __( 'How long store content stays fresh before it is refreshed (default: 60). Expired content is still shown instantly while it refreshes in the background, so raising this does not slow your pages down.', 'fourthwall_text_domain' ) . '

'; } diff --git a/libs/shortcode.php b/libs/shortcode.php index 8ebe3cc..a223840 100644 --- a/libs/shortcode.php +++ b/libs/shortcode.php @@ -1,34 +1,154 @@ time(); +} + +/** + * Queue a background refresh for a stale URL. + * + * WP-Cron fires on a later request, so the visitor who found the stale entry is + * never the one who waits for the network. + * + * @param string $url The URL to refresh + * @return void + */ +function fwembed_schedule_refresh($url) { + if (!function_exists('wp_next_scheduled') || !function_exists('wp_schedule_single_event')) { + return; + } + + // Collapses a stampede: many visitors hitting the same stale entry queue + // one job between them. + if (wp_next_scheduled('fwembed_refresh_url', array($url))) { + return; + } + + wp_schedule_single_event(time() + 1, 'fwembed_refresh_url', array($url)); +} + +/** + * WP-Cron handler: re-fetch a URL and replace its cache entry. + * + * @param string $url The URL to refresh + * @return void + */ +function fwembed_do_refresh($url) { + $lock_key = fwembed_cache_key($url) . '_lock'; + + // Guards against two overlapping cron runs fetching the same URL. + if (get_transient($lock_key) !== false) { + return; + } + set_transient($lock_key, 1, 2 * MINUTE_IN_SECONDS); + + $result = fwembed_fetch($url); + if (!$result['error'] && $result['status_code'] == 200) { + fwembed_cache_write($url, $result); + } + + delete_transient($lock_key); +} +add_action('fwembed_refresh_url', 'fwembed_do_refresh'); + +/** + * Build a configured cURL handle. + * + * @param string $url The URL to fetch + * @return resource|CurlHandle + */ +function fwembed_curl_handle($url) { // More complete browser-like headers - $headers = [ + $headers = array( 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Accept-Language: en-US,en;q=0.5', 'Connection: keep-alive', 'Upgrade-Insecure-Requests: 1', 'Cache-Control: max-age=0' - ]; + ); + $ch = curl_init(); curl_setopt($ch, CURLOPT_URL, $url); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); @@ -36,81 +156,304 @@ function fwembed_make_request($url) { curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0'); curl_setopt($ch, CURLOPT_TIMEOUT, 30); - curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, $ssl_verify); - curl_setopt($ch, CURLOPT_COOKIEJAR, '/tmp/cookies.txt'); - curl_setopt($ch, CURLOPT_COOKIEFILE, '/tmp/cookies.txt'); + curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10); + curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, true); + curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 2); - $html_content = curl_exec($ch); + return $ch; +} + +/** + * Turn a finished cURL handle into a response array. + * + * @param resource|CurlHandle $ch The handle + * @param string|bool $content The transferred body + * @return array Response with 'content', 'error' and 'status_code' + */ +function fwembed_curl_result($ch, $content) { $error = null; $status_code = 0; if (curl_errno($ch)) { $error = curl_error($ch); } else { - $status_code = curl_getinfo($ch, CURLINFO_HTTP_CODE); + $status_code = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE); if ($status_code == 403) { $error = "Access forbidden (403). The website may be blocking automated requests."; + } elseif ($status_code >= 400) { + $error = "Request failed with HTTP status " . $status_code . "."; } } + return array( + 'content' => $content, + 'error' => $error, + 'status_code' => $status_code, + ); +} + +/** + * Fetch a single URL, bypassing the cache. + * + * @param string $url The URL to fetch + * @return array Response array + */ +function fwembed_fetch($url) { + $ch = fwembed_curl_handle($url); + $content = curl_exec($ch); + $result = fwembed_curl_result($ch, $content); curl_close($ch); - $result = [ - 'content' => $html_content, - 'error' => $error, - 'status_code' => $status_code - ]; - - // Cache the result for 1 hour (3600 seconds) if no error - if (!$error && $status_code == 200) { - set_transient($cache_key, $result, 3600); - } - return $result; } +/** + * Fetch several URLs concurrently, bypassing the cache. + * + * @param array $urls URLs to fetch + * @param int $concurrency Maximum simultaneous connections + * @return array Map of URL => response array + */ +function fwembed_fetch_many($urls, $concurrency = 8) { + $urls = array_values(array_unique($urls)); + $results = array(); + + if (empty($urls)) { + return $results; + } + if (count($urls) === 1 || !function_exists('curl_multi_init')) { + foreach ($urls as $url) { + $results[$url] = fwembed_fetch($url); + } + return $results; + } + + foreach (array_chunk($urls, max(1, (int) $concurrency)) as $batch) { + $multi = curl_multi_init(); + $handles = array(); + + foreach ($batch as $url) { + $ch = fwembed_curl_handle($url); + curl_multi_add_handle($multi, $ch); + $handles[$url] = $ch; + } + + $running = null; + do { + $status = curl_multi_exec($multi, $running); + if ($running) { + curl_multi_select($multi, 1.0); + } + } while ($running && $status === CURLM_OK); + + foreach ($handles as $url => $ch) { + $results[$url] = fwembed_curl_result($ch, curl_multi_getcontent($ch)); + curl_multi_remove_handle($multi, $ch); + curl_close($ch); + } + + curl_multi_close($multi); + } + + return $results; +} + +/** + * Common function to make HTTP requests to Fourthwall + * + * Serves cached content immediately whenever any is stored. A stale entry is + * returned as-is and refreshed by WP-Cron, so expiry never lands the cost of a + * network round trip on a visitor. + * + * @param string $url The URL to fetch + * @return array Array containing 'content' and 'error' keys + */ +function fwembed_make_request($url) { + $cached = fwembed_cache_read($url); + + if ($cached !== null) { + if (!fwembed_cache_is_fresh($cached)) { + fwembed_schedule_refresh($url); + } + return $cached; + } + + $result = fwembed_fetch($url); + + if (!$result['error'] && $result['status_code'] == 200) { + $result = fwembed_cache_write($url, $result); + } + + return $result; +} + +/** + * Make several requests at once, using the cache where possible. + * + * Anything not already cached is fetched in parallel, turning N sequential + * round trips into one batch. + * + * @param array $urls URLs to fetch + * @return array Map of URL => response array + */ +function fwembed_make_requests($urls) { + $results = array(); + $pending = array(); + + foreach (array_unique($urls) as $url) { + $cached = fwembed_cache_read($url); + + if ($cached !== null) { + if (!fwembed_cache_is_fresh($cached)) { + fwembed_schedule_refresh($url); + } + $results[$url] = $cached; + continue; + } + + $pending[] = $url; + } + + foreach (fwembed_fetch_many($pending) as $url => $result) { + if (!$result['error'] && $result['status_code'] == 200) { + $result = fwembed_cache_write($url, $result); + } + $results[$url] = $result; + } + + return $results; +} + +/** + * Build an XPath predicate matching a single CSS class name. + * + * Fourthwall reorders and appends classes between releases, so never match + * @class exactly - pad the attribute and look for the padded class name. + * + * @param string $class The class name to match + * @return string XPath predicate fragment + */ +function fwembed_class_predicate($class) { + return 'contains(concat(" ", normalize-space(@class), " "), " ' . $class . ' ")'; +} + +/** + * Run XPath queries in priority order and return the first one that matches. + * + * Lets each selector degrade to a looser fallback when Fourthwall changes + * their markup, instead of the whole product silently disappearing. + * + * @param DOMXPath $xpath The XPath engine + * @param array $queries XPath expressions, most specific first + * @param DOMNode|null $context Optional context node for relative queries + * @return DOMNodeList|null First non-empty result, or null if none matched + */ +function fwembed_query_first(DOMXPath $xpath, array $queries, $context = null) { + foreach ($queries as $query) { + $nodes = $context === null ? $xpath->query($query) : $xpath->query($query, $context); + if ($nodes !== false && $nodes->length > 0) { + return $nodes; + } + } + return null; +} + +/** + * Resolve a possibly-relative href against the page it was found on. + * + * Product hrefs are root-relative ("/products/foo"), so naive concatenation + * breaks whenever the store URL has a trailing slash or a path such as + * /collections/all. + * + * @param string $href The href to resolve + * @param string $page_url The URL of the page the href was found on + * @return string Absolute URL + */ +function fwembed_resolve_url($href, $page_url) { + $href = trim($href); + if ($href === '') { + return ''; + } + if (preg_match('#^https?://#i', $href)) { + return $href; + } + + $parts = parse_url($page_url); + if (empty($parts['scheme']) || empty($parts['host'])) { + return $href; + } + + $origin = $parts['scheme'] . '://' . $parts['host']; + if (!empty($parts['port'])) { + $origin .= ':' . $parts['port']; + } + + if (strpos($href, '//') === 0) { + return $parts['scheme'] . ':' . $href; + } + if (strpos($href, '/') === 0) { + return $origin . $href; + } + + $path = isset($parts['path']) ? $parts['path'] : '/'; + $dir = (substr($path, -1) === '/') ? $path : str_replace('\\', '/', dirname($path)) . '/'; + + return $origin . $dir . $href; +} + /** * Parse HTML content and extract product tiles - * + * * @param string $html_content The HTML content to parse * @param string $base_url The base URL for constructing links * @return string Parsed HTML content */ function fwembed_parse_product_tiles($html_content, $base_url) { - $html = null; + $html = ''; libxml_use_internal_errors(true); $dom = new DOMDocument(); @$dom->loadHTML(loadHTML5($html_content), LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD); $dom->documentURI = $base_url; - $divs = $dom->getElementsByTagName('div'); - - foreach ($divs as $div) { - if ($div->hasAttribute('data-testid') && $div->getAttribute('data-testid') === 'product') { - $xpath = new DOMXPath($dom); + $xpath = new DOMXPath($dom); - $tileLink = $xpath->query('.//a[contains(@class, "tile")]', $div); - $tileItem = $xpath->query('.//img[contains(@class, "tile__item--1") - and not(contains(@class, "badge")) - and not(contains(@class, "tile_options")) - and not(contains(@class, "tile__item--2"))]', $div); - $tileDesc = $xpath->query('.//*[contains(@class, "tile__description") - and not(contains(@class, "badge")) - and not(contains(@class, "tile_options"))]', $div); - - $productHTML = ''; - $linkHref = ''; - if ($tileLink->length > 0) { - $linkHref = $tileLink->item(0)->getAttribute('href'); - } - if ($tileItem->length > 0) { - $productHTML .= $dom->saveHTML($tileItem->item(0)); - } - if ($tileDesc->length > 0) { - $productHTML .= $dom->saveHTML($tileDesc->item(0)); - } + $products = $xpath->query('//*[@data-testid="product"]'); + if ($products === false || $products->length === 0) { + $products = $xpath->query('//*[' . fwembed_class_predicate('product-tile') . ']'); + } - $html = $html . '
' . $productHTML . '
'; + foreach ($products as $product) { + $tileLink = fwembed_query_first($xpath, array( + './/a[' . fwembed_class_predicate('tile') . ']', + './/a[@href]', + ), $product); + + $tileItem = fwembed_query_first($xpath, array( + './/img[' . fwembed_class_predicate('tile__item--1') . ']', + './/*[' . fwembed_class_predicate('image__object') . ']//img', + './/img', + ), $product); + + $tileDesc = fwembed_query_first($xpath, array( + './/*[' . fwembed_class_predicate('tile__description') . ']', + ), $product); + + $productHTML = ''; + $linkHref = ''; + if ($tileLink !== null) { + $linkHref = fwembed_resolve_url($tileLink->item(0)->getAttribute('href'), $base_url); } + if ($tileItem !== null) { + $productHTML .= $dom->saveHTML($tileItem->item(0)); + } + if ($tileDesc !== null) { + $productHTML .= $dom->saveHTML($tileDesc->item(0)); + } + + if ($productHTML === '') { + continue; + } + + $html .= '
' . $productHTML . '
'; } libxml_clear_errors(); return $html; @@ -142,63 +485,376 @@ function fwembed_shortcode( $atts ) { return $store_render; } +/** + * Decode every JSON-LD block on a page. + * + * @param DOMXPath $xpath The XPath engine + * @return array List of decoded JSON-LD structures + */ +function fwembed_jsonld_blocks(DOMXPath $xpath) { + $blocks = array(); + $scripts = $xpath->query('//script[contains(@type, "ld+json")]'); + if ($scripts === false) { + return $blocks; + } + + foreach ($scripts as $script) { + $decoded = json_decode(trim($script->textContent), true); + if (json_last_error() === JSON_ERROR_NONE && is_array($decoded)) { + $blocks[] = $decoded; + } + } + return $blocks; +} + +/** + * Walk a decoded JSON-LD structure looking for a schema.org Product node. + * + * Handles the shapes publishers actually emit: a bare object, a list, or a + * node wrapped in @graph / itemListElement / mainEntity. + * + * @param mixed $data Decoded JSON-LD fragment + * @param int $depth Current recursion depth + * @return array|null The Product node, or null if this branch has none + */ +function fwembed_find_jsonld_product($data, $depth = 0) { + if (!is_array($data) || $depth > 6) { + return null; + } + + if (isset($data['@type'])) { + foreach ((array) $data['@type'] as $type) { + if (is_string($type) && strcasecmp($type, 'Product') === 0) { + return $data; + } + } + } + + foreach (array('@graph', 'itemListElement', 'mainEntity', 'item') as $key) { + if (isset($data[$key])) { + $found = fwembed_find_jsonld_product($data[$key], $depth + 1); + if ($found !== null) { + return $found; + } + } + } + + // A plain list of nodes. + foreach ($data as $key => $item) { + if (is_int($key) && is_array($item)) { + $found = fwembed_find_jsonld_product($item, $depth + 1); + if ($found !== null) { + return $found; + } + } + } + + return null; +} + +/** + * Pick the cheapest offer from a JSON-LD Product node. + * + * Variants each get their own Offer, so the lowest price is what a storefront + * shows as the headline figure. AggregateOffer states it directly as lowPrice. + * + * @param array $product A schema.org Product node + * @return array Keys 'price', 'currency', 'availability'; empty if no offer + */ +function fwembed_jsonld_offer($product) { + if (empty($product['offers'])) { + return array(); + } + + $offers = $product['offers']; + if (isset($offers['@type']) || isset($offers['price']) || isset($offers['lowPrice'])) { + $offers = array($offers); + } + + $best = array(); + foreach ((array) $offers as $offer) { + if (!is_array($offer)) { + continue; + } + foreach (array('lowPrice', 'price') as $field) { + if (!isset($offer[$field]) || !is_numeric($offer[$field])) { + continue; + } + $price = (float) $offer[$field]; + if (empty($best) || $price < $best['price']) { + $availability = isset($offer['availability']) ? (string) $offer['availability'] : ''; + $best = array( + 'price' => $price, + 'currency' => isset($offer['priceCurrency']) ? (string) $offer['priceCurrency'] : '', + // "https://schema.org/InStock" -> "InStock" + 'availability' => $availability === '' ? '' : substr($availability, strrpos($availability, '/') + 1), + ); + } + break; + } + } + return $best; +} + +/** + * Format a numeric price for display. + * + * Only used when the storefront's own formatted price could not be scraped. + * + * @param float $amount The price + * @param string $currency ISO 4217 currency code + * @return string Display-ready price + */ +function fwembed_format_price($amount, $currency) { + $currency = strtoupper(trim($currency)); + $symbols = array( + 'USD' => '$', 'CAD' => '$', 'AUD' => '$', 'NZD' => '$', + 'EUR' => "\xE2\x82\xAC", 'GBP' => "\xC2\xA3", 'JPY' => "\xC2\xA5", + ); + + $formatted = number_format($amount, $currency === 'JPY' ? 0 : 2); + + if (isset($symbols[$currency])) { + return $symbols[$currency] . $formatted; + } + return $currency === '' ? $formatted : $currency . ' ' . $formatted; +} + +/** + * Extract product fields from a page's JSON-LD. + * + * Preferred over scraping: it is a published contract rather than styling, so + * it survives the theme changes that repeatedly break CSS-class selectors. + * + * @param DOMXPath $xpath The XPath engine + * @return array Normalised product fields (any may be absent) + */ +function fwembed_product_from_jsonld(DOMXPath $xpath) { + $product = null; + foreach (fwembed_jsonld_blocks($xpath) as $block) { + $product = fwembed_find_jsonld_product($block); + if ($product !== null) { + break; + } + } + if ($product === null) { + return array(); + } + + $fields = array(); + + if (!empty($product['name']) && is_string($product['name'])) { + $fields['title'] = $product['name']; + } + + if (!empty($product['image'])) { + $image = $product['image']; + // May be a string, a list, or an ImageObject. + if (is_array($image)) { + $image = isset($image[0]) ? $image[0] : (isset($image['url']) ? $image['url'] : null); + } + if (is_array($image) && isset($image['url'])) { + $image = $image['url']; + } + if (is_string($image) && $image !== '') { + $fields['image'] = $image; + } + } + + if (!empty($product['description']) && is_string($product['description'])) { + $fields['description_text'] = $product['description']; + } + + if (!empty($product['sku']) && is_string($product['sku'])) { + $fields['sku'] = $product['sku']; + } + + $offer = fwembed_jsonld_offer($product); + if (!empty($offer)) { + $fields['price'] = fwembed_format_price($offer['price'], $offer['currency']); + $fields['currency'] = $offer['currency']; + $fields['availability'] = $offer['availability']; + } + + return $fields; +} + +/** + * Extract product fields by scraping the rendered markup. + * + * Fallback for stores whose JSON-LD is missing or incomplete, and the source of + * the store's own price formatting and rich HTML description. + * + * @param DOMDocument $dom The parsed page + * @param DOMXPath $xpath The XPath engine + * @param string $url The product URL + * @return array Normalised product fields (any may be absent) + */ +function fwembed_product_from_html(DOMDocument $dom, DOMXPath $xpath, $url) { + $fields = array(); + + // Fourthwall moved the title from

to

, so match on the class / + // data-testid rather than the tag, and fall back to the OpenGraph tags + // (which have stayed stable across their redesigns). + $productTitle = fwembed_query_first($xpath, array( + '//*[@data-testid="product.name"]', + '//*[' . fwembed_class_predicate('product-info__title') . ']', + '//meta[@property="og:title"]', + )); + $productPrice = fwembed_query_first($xpath, array( + '//*[@data-testid="product.price"]', + '//*[' . fwembed_class_predicate('product-info__price--original') . ']', + '//*[' . fwembed_class_predicate('product-info__price') . ']', + )); + $productImage = fwembed_query_first($xpath, array( + '//*[@data-testid="product.image"]//img[' . fwembed_class_predicate('gallery__image-object') . ']', + '//*[@data-gallery="gallery-slide"]//img[' . fwembed_class_predicate('gallery__image-object') . ']', + '//img[' . fwembed_class_predicate('gallery__image-object') . ']', + '//meta[@property="og:image"]', + )); + $productDesc = fwembed_query_first($xpath, array( + '//*[' . fwembed_class_predicate('product-info__description') . ']//*[' . fwembed_class_predicate('html-formatter') . ']', + '//*[' . fwembed_class_predicate('product-info__description') . ']', + )); + + if ($productTitle !== null) { + $node = $productTitle->item(0); + $fields['title'] = $node->nodeName === 'meta' ? $node->getAttribute('content') : $node->textContent; + } + + if ($productPrice !== null) { + $fields['price'] = $productPrice->item(0)->textContent; + } + + if ($productImage !== null) { + $node = $productImage->item(0); + if ($node->nodeName === 'meta') { + $src = $node->getAttribute('content'); + } else { + // Lazy-loaded galleries keep the real URL in data-src. + $src = $node->getAttribute('src'); + if ($src === '' || strpos($src, 'data:') === 0) { + $src = $node->getAttribute('data-src'); + } + $alt = $node->getAttribute('alt'); + if ($alt !== '') { + $fields['image_alt'] = $alt; + } + } + if (trim($src) !== '') { + $fields['image'] = fwembed_resolve_url($src, $url); + } + } + + if ($productDesc !== null) { + $fields['description_html'] = $dom->saveHTML($productDesc->item(0)); + } + + return $fields; +} + /** * Parse HTML content and extract single product information - * + * + * Reads JSON-LD first and falls back to scraping field by field, so a markup + * change can only degrade whatever it actually touched. + * * @param string $html_content The HTML content to parse * @param string $url The product URL * @param bool $show_description Whether to show product description * @return string Parsed HTML content */ function fwembed_parse_single_product($html_content, $url, $show_description = false) { - $html = null; + $html = ''; libxml_use_internal_errors(true); $dom = new DOMDocument(); @$dom->loadHTML(loadHTML5($html_content), LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD); $dom->documentURI = $url; $xpath = new DOMXPath($dom); - - // Extract product information - $productTitle = $xpath->query('//h2[@class="product-info__title"]'); - $productPrice = $xpath->query('//span[@class="product-info__price product-info__price--original"]'); - $productImage = $xpath->query('//div[@data-gallery="gallery-slide"][1]//img[@class="gallery__image-object"]'); - $productDesc = $xpath->query('//div[@class="product-info__description"]//div[@class="html-formatter"]'); - - if ($productTitle->length > 0 && $productImage->length > 0) { - $title = $productTitle->item(0)->textContent; - $price = $productPrice->length > 0 ? $productPrice->item(0)->textContent : ''; - - // Get image attributes - $imageNode = $productImage->item(0); - $imageSrc = $imageNode->getAttribute('src'); - $imageAlt = $imageNode->getAttribute('alt'); - - // Build the HTML - $html = '
'; - $html .= ''; - $html .= '' . htmlspecialchars($imageAlt) . ''; - - $html .= '
'; - $html .= '

' . htmlspecialchars(trim($title)) . '

'; - $html .= '
'; - $html .= '' . trim($price) . ''; - $html .= '
'; - $html .= '
'; - $html .= ''; - - // Add description if show_description is true - if ($show_description && $productDesc->length > 0) { - $description = $dom->saveHTML($productDesc->item(0)); - $html .= '
' . $description . '
'; - } - - $html .= '
'; - } - + + $structured = fwembed_product_from_jsonld($xpath); + $scraped = fwembed_product_from_html($dom, $xpath, $url); + + // JSON-LD wins where it is the more stable source. The scraped price wins + // because it carries the store's own formatting ("from $20", locale rules), + // and the scraped description wins because JSON-LD flattens it to plain text. + $title = fwembed_first_value(array( + isset($structured['title']) ? $structured['title'] : null, + isset($scraped['title']) ? $scraped['title'] : null, + )); + $image = fwembed_first_value(array( + isset($structured['image']) ? $structured['image'] : null, + isset($scraped['image']) ? $scraped['image'] : null, + )); + $price = fwembed_first_value(array( + isset($scraped['price']) ? $scraped['price'] : null, + isset($structured['price']) ? $structured['price'] : null, + )); + libxml_clear_errors(); + + if ($title === '' || $image === '') { + return ''; + } + + $image = fwembed_resolve_url($image, $url); + $imageAlt = isset($scraped['image_alt']) ? $scraped['image_alt'] : $title; + + $availability = isset($structured['availability']) ? $structured['availability'] : ''; + $classes = 'product-tile'; + if (strcasecmp($availability, 'OutOfStock') === 0 || strcasecmp($availability, 'SoldOut') === 0) { + $classes .= ' product-tile--sold-out'; + } + + $attributes = ''; + if ($availability !== '') { + $attributes .= ' data-availability="' . esc_attr($availability) . '"'; + } + if (!empty($structured['sku'])) { + $attributes .= ' data-sku="' . esc_attr($structured['sku']) . '"'; + } + + $html = '
'; + $html .= ''; + $html .= '' . esc_attr($imageAlt) . ''; + + $html .= '
'; + $html .= '

' . esc_html(trim($title)) . '

'; + $html .= '
'; + $html .= '' . esc_html(trim($price)) . ''; + $html .= '
'; + $html .= '
'; + $html .= ''; + + // Add description if show_description is true + if ($show_description) { + if (isset($scraped['description_html'])) { + $html .= '
' . $scraped['description_html'] . '
'; + } elseif (isset($structured['description_text'])) { + $html .= '
' . wpautop(esc_html($structured['description_text'])) . '
'; + } + } + + $html .= '
'; + return $html; } +/** + * Return the first non-empty string from a list of candidates. + * + * @param array $candidates Values in priority order + * @return string First usable value, or an empty string + */ +function fwembed_first_value($candidates) { + foreach ($candidates as $candidate) { + if (is_string($candidate) && trim($candidate) !== '') { + return $candidate; + } + } + return ''; +} + function fwembed_parse_html_single($url = null, $show_description = false) { if ($url === null) { throw new ValueError("Missing URL"); @@ -249,66 +905,174 @@ function fwembed_extract_product_urls($html_content, $base_url) { $dom = new DOMDocument(); @$dom->loadHTML(loadHTML5($html_content), LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD); $dom->documentURI = $base_url; - $divs = $dom->getElementsByTagName('div'); - - foreach ($divs as $div) { - if ($div->hasAttribute('data-testid') && $div->getAttribute('data-testid') === 'product') { - $xpath = new DOMXPath($dom); - $tileLink = $xpath->query('.//a[contains(@class, "tile")]', $div); - - if ($tileLink->length > 0) { - $linkHref = $tileLink->item(0)->getAttribute('href'); - $full_url = $base_url . $linkHref; + $xpath = new DOMXPath($dom); + + $products = $xpath->query('//*[@data-testid="product"]'); + if ($products === false || $products->length === 0) { + $products = $xpath->query('//*[' . fwembed_class_predicate('product-tile') . ']'); + } + + foreach ($products as $product) { + $tileLink = fwembed_query_first($xpath, array( + './/a[' . fwembed_class_predicate('tile') . ']', + './/a[@href]', + ), $product); + + if ($tileLink !== null) { + $full_url = fwembed_resolve_url($tileLink->item(0)->getAttribute('href'), $base_url); + if ($full_url !== '') { $product_urls[] = $full_url; } } } libxml_clear_errors(); - return $product_urls; + + // The same product can appear in several sections of one page. + return array_values(array_unique($product_urls)); +} + +/** + * Read product URLs from a sitemap, following one level of sitemap index. + * + * @param string $sitemap_url The sitemap to read + * @param string $store_url The store URL, used to reject off-site entries + * @param int $depth Current recursion depth + * @return array Product URLs + */ +function fwembed_read_sitemap($sitemap_url, $store_url, $depth = 0) { + if ($depth > 1) { + return array(); + } + + $result = fwembed_make_request($sitemap_url); + if ($result['error'] || empty($result['content'])) { + return array(); + } + + libxml_use_internal_errors(true); + $xml = new DOMDocument(); + $loaded = @$xml->loadXML($result['content']); + if (!$loaded || $xml->documentElement === null) { + libxml_clear_errors(); + return array(); + } + + $store_host = parse_url($store_url, PHP_URL_HOST); + $is_index = strcasecmp($xml->documentElement->localName, 'sitemapindex') === 0; + $urls = array(); + $nested = 0; + + foreach ($xml->getElementsByTagName('loc') as $loc) { + $value = trim($loc->textContent); + if ($value === '') { + continue; + } + + // Never follow a pointing off the configured store. + if (parse_url($value, PHP_URL_HOST) !== $store_host) { + continue; + } + + if ($is_index) { + if ($nested >= 10) { + break; + } + $nested++; + $urls = array_merge($urls, fwembed_read_sitemap($value, $store_url, $depth + 1)); + } elseif (strpos($value, '/products/') !== false) { + $urls[] = $value; + } + } + + libxml_clear_errors(); + return array_values(array_unique($urls)); +} + +/** + * List every product in a store via its sitemap. + * + * The store page only renders whatever collection it is configured to feature, + * so scraping it under-reports the catalogue. The sitemap is the full list. + * + * @param string $store_url The store URL + * @return array Product URLs + */ +function fwembed_extract_product_urls_from_sitemap($store_url) { + $sitemap_url = fwembed_resolve_url('/sitemap.xml', $store_url); + if ($sitemap_url === '') { + return array(); + } + return fwembed_read_sitemap($sitemap_url, $store_url); +} + +/** + * Collect candidate product URLs for a store. + * + * @param string $store_url The store URL + * @param string $source One of 'auto', 'sitemap', 'page' + * @return array|string Product URLs, or an error string + */ +function fwembed_collect_product_urls($store_url, $source = 'auto') { + if ($source !== 'page') { + $urls = fwembed_extract_product_urls_from_sitemap($store_url); + if (!empty($urls) || $source === 'sitemap') { + return $urls; + } + } + + $result = fwembed_make_request($store_url); + if ($result['error']) { + return "Error fetching store URL: " . $result['error']; + } + + return fwembed_extract_product_urls($result['content'], $store_url); } /** * Get random products from store or specified URLs - * + * * @param string $store_url The store URL to fetch products from (optional if urls provided) * @param array $urls Array of specific product URLs to randomize from * @param int $count Number of products to display + * @param string $source Where to discover products: 'auto', 'sitemap' or 'page' * @return string HTML content of random products */ -function fwembed_get_random_products($store_url = null, $urls = array(), $count = 3) { +function fwembed_get_random_products($store_url = null, $urls = array(), $count = 3, $source = 'auto') { $product_urls = array(); - + // If specific URLs are provided, use those if (!empty($urls)) { $product_urls = $urls; - } - // Otherwise, fetch all products from the store + } + // Otherwise, discover all products in the store elseif ($store_url) { - $result = fwembed_make_request($store_url); - - if ($result['error']) { - return "Error fetching store URL: " . $result['error']; + $product_urls = fwembed_collect_product_urls($store_url, $source); + if (is_string($product_urls)) { + return $product_urls; } - - $product_urls = fwembed_extract_product_urls($result['content'], $store_url); } else { return "Error: Either store URL or product URLs must be provided"; } - + // Shuffle the URLs to randomize shuffle($product_urls); // Limit to requested count $selected_urls = array_slice($product_urls, 0, $count); + // Fetch every uncached product page in one parallel batch rather than + // serially, so a cold cache costs one round trip instead of N. + $responses = fwembed_make_requests($selected_urls); + $html = ''; foreach ($selected_urls as $url) { - $product_html = fwembed_parse_html_single($url, false); - if ($product_html && !strpos($product_html, 'Error fetching URL')) { - $html .= $product_html; + if (empty($responses[$url]) || $responses[$url]['error'] || empty($responses[$url]['content'])) { + continue; } + + $html .= fwembed_parse_single_product($responses[$url]['content'], $url, false); } - + return $html; } @@ -318,10 +1082,16 @@ function fwembed_random_shortcode($atts) { 'count' => '3', 'urls' => '', 'store_url' => '', + 'source' => 'auto', ), $atts ); - + + $source = strtolower(trim($atts['source'])); + if (!in_array($source, array('auto', 'sitemap', 'page'), true)) { + $source = 'auto'; + } + $count = intval($atts['count']); if ($count <= 0) { $count = 3; @@ -348,7 +1118,7 @@ function fwembed_random_shortcode($atts) { $store_url = isset($options['fourth_url']) ? $options['fourth_url'] : ''; } - $products_html = fwembed_get_random_products($store_url, $urls, $count); + $products_html = fwembed_get_random_products($store_url, $urls, $count, $source); if (empty($products_html)) { return '

No products found to display.

';