Use CORS proxy services

sapols · sapols · commit b9c1d3502246 · 2025-03-06T15:40:17.000-07:00
diff --git a/README.md b/README.md
@@ -14,7 +14,20 @@ The PyHC Documentation Hub allows searching across documentation for all PyHC pa
 
 ## How It Works
 
-The PyHC Documentation Hub leverages the Read The Docs Server-Side Search API to query multiple project documentations simultaneously. The search is performed client-side using JavaScript, which constructs queries to the Read The Docs API and presents the results in a unified interface.
+The PyHC Documentation Hub leverages the Read The Docs Server-Side Search API to query multiple project documentations simultaneously. The search is performed client-side using JavaScript, which constructs queries to the Read The Docs API through a CORS proxy. The results are presented in a unified interface.
+
+### CORS Handling
+
+Due to CORS restrictions, browsers will block direct requests from client-side JavaScript to the Read The Docs API. To solve this, we use public CORS proxy services to add the appropriate CORS headers to the API responses.
+
+The search functionality uses multiple CORS proxy services (in order of preference):
+1. corsproxy.io
+2. cors-anywhere.herokuapp.com
+3. allorigins.win
+
+If one proxy fails, the application automatically tries the next one. This provides redundancy and ensures the search functionality works consistently across different browsers and environments.
+
+The implementation can be found in `source/_static/pyhc_search.js`.
 
 ## Development
 
@@ -32,6 +45,16 @@ The PyHC Documentation Hub leverages the Read The Docs Server-Side Search API to
 3. Build the documentation: `make html`
 4. Open `build/html/index.html` in your browser
 
+For development, you can use Python's built-in HTTP server:
+
+```bash
+# Build and serve the documentation
+make html
+python -m http.server -d build/html
+```
+
+Then access the documentation at http://localhost:8000
+
 ## License
 
 This project is licensed under the terms of the LICENSE file included in this repository.
diff --git a/requirements.txt b/requirements.txt
@@ -0,0 +1,3 @@
+sphinx==7.2.6
+sphinx-rtd-theme==2.0.0
+sphinx-copybutton==0.5.2
diff --git a/source/_static/pyhc_search.js b/source/_static/pyhc_search.js
@@ -1,4 +1,6 @@
 // PyHC Documentation Hub - Unified Search
+// This script uses public CORS proxy services to make cross-origin requests to the Read the Docs API.
+// Multiple proxy services are tried in sequence for redundancy.
 document.addEventListener('DOMContentLoaded', function() {
   // Add scroll event handler to toggle shadow on tabs when scrolling
   function initScrollShadow() {
@@ -28,8 +30,29 @@ document.addEventListener('DOMContentLoaded', function() {
   const rtdSearchForm = document.getElementById('rtd-search-form');
   const rtdSearchInput = rtdSearchForm ? rtdSearchForm.querySelector('input[type="text"]') : null;
   
-  // Base API URL
-  const apiBaseUrl = 'https://readthedocs.org/api/v3/search/';
+  // Base API URL with public CORS proxy services
+  const rtdApiUrl = 'https://readthedocs.org/api/v3/search/';
+  
+  // List of CORS proxies to try (in order of preference)
+  const corsProxies = [
+    'https://corsproxy.io/?',
+    'https://cors-anywhere.herokuapp.com/',
+    'https://api.allorigins.win/raw?url='
+  ];
+  
+  // Start with the first proxy
+  let currentProxyIndex = 0;
+  const corsProxyUrl = corsProxies[currentProxyIndex];
+  let apiBaseUrl = corsProxyUrl + encodeURIComponent(rtdApiUrl);
+  
+  // Function to switch to the next proxy if one fails
+  function switchToNextProxy() {
+    currentProxyIndex = (currentProxyIndex + 1) % corsProxies.length;
+    const nextProxy = corsProxies[currentProxyIndex];
+    apiBaseUrl = nextProxy + encodeURIComponent(rtdApiUrl);
+    console.log(`Switched to CORS proxy: ${nextProxy}`);
+    return apiBaseUrl;
+  }
   
   // Modal elements (will be defined after createSearchModal)
   const searchModal = document.getElementById('pyhc-search-modal');
@@ -446,8 +469,11 @@ document.addEventListener('DOMContentLoaded', function() {
     // Build the full search query with space between project list and search term
     const fullQuery = `${projectsQuery} ${query}`;
     
-    // Build the URL with the correctly encoded query parameter
-    const queryUrl = `${apiBaseUrl}?q=${encodeURIComponent(fullQuery)}`;
+    // Build the query parameter
+    const queryParam = `q=${encodeURIComponent(fullQuery)}`;
+    
+    // When using corsproxy.io, we need to add the query parameter to the encoded URL
+    const queryUrl = `${apiBaseUrl}${encodeURIComponent('?' + queryParam)}`;
     
     // Fetch results
     fetch(queryUrl)
@@ -463,9 +489,43 @@ document.addEventListener('DOMContentLoaded', function() {
         displayResults(data, query);
       })
       .catch(error => {
-        // Reset search icon to magnifying glass
-        resetSearchIcon();
-        showMessage(`An error occurred while searching: ${error.message}. Please try again later.`);
+        console.error("Search error:", error);
+        
+        // Check if we've tried all proxies
+        if (corsProxies.length > 1) {
+          // Try with next proxy
+          switchToNextProxy();
+          
+          // Update the query URL with the new proxy
+          const newQueryUrl = `${apiBaseUrl}${encodeURIComponent('?' + queryParam)}`;
+          
+          // Show a trying alternative proxy message
+          showMessage(`Trying alternative proxy service... Please wait.`);
+          
+          // Retry the search with the new proxy
+          setTimeout(() => {
+            fetch(newQueryUrl)
+              .then(response => {
+                if (!response.ok) {
+                  throw new Error(`Network response was not ok: ${response.status} ${response.statusText}`);
+                }
+                return response.json();
+              })
+              .then(data => {
+                resetSearchIcon();
+                displayResults(data, query);
+              })
+              .catch(retryError => {
+                // If all proxies fail, show error message
+                resetSearchIcon();
+                showMessage(`Search failed. The search service may be temporarily unavailable. Please try again later.`);
+              });
+          }, 1000);
+        } else {
+          // Reset search icon to magnifying glass
+          resetSearchIcon();
+          showMessage(`An error occurred while searching: ${error.message}. Please try again later.`);
+        }
       });
   }
   
@@ -869,7 +929,10 @@ document.addEventListener('DOMContentLoaded', function() {
       loadMoreBtn.disabled = true;
     }
     
-    fetch(nextUrl)
+    // We need to handle the next URL through the CORS proxy
+    const proxiedNextUrl = corsProxyUrl + encodeURIComponent(nextUrl);
+    
+    fetch(proxiedNextUrl)
       .then(response => {
         if (!response.ok) {
           throw new Error('Network response was not ok');
@@ -981,9 +1044,142 @@ document.addEventListener('DOMContentLoaded', function() {
       })
       .catch(error => {
         console.error('Error loading more results:', error);
-        if (loadMoreBtn) {
-          loadMoreBtn.textContent = 'Error loading more results. Click to try again.';
-          loadMoreBtn.disabled = false;
+        
+        // Try with the next proxy if available
+        if (corsProxies.length > 1) {
+          switchToNextProxy();
+          const retryProxiedUrl = corsProxyUrl + encodeURIComponent(nextUrl);
+          
+          if (loadMoreBtn) {
+            loadMoreBtn.textContent = 'Trying alternative proxy service...';
+          }
+          
+          // Retry with new proxy
+          setTimeout(() => {
+            fetch(retryProxiedUrl)
+              .then(response => {
+                if (!response.ok) {
+                  throw new Error('Network response was not ok');
+                }
+                return response.json();
+              })
+              .then(data => {
+                // Remove the load more button
+                if (loadMoreBtn) {
+                  loadMoreBtn.remove();
+                }
+                
+                // Continue with processing the data as before
+                // Group new results by project
+                const resultsByProject = {};
+                
+                data.results.forEach(result => {
+                  const projectSlug = result.project.slug;
+                  if (!resultsByProject[projectSlug]) {
+                    resultsByProject[projectSlug] = [];
+                  }
+                  resultsByProject[projectSlug].push(result);
+                });
+                
+                // Get existing project tabs container
+                const tabsContainer = resultsContainer.querySelector('.project-tabs');
+                
+                // Process each project's results
+                for (const [project, results] of Object.entries(resultsByProject)) {
+                  // Check if there's already a container for this project
+                  let projectContainer = resultsContainer.querySelector(`.project-results[data-project="${project}"]`);
+                  
+                  // If there's no tab for this project yet, create it
+                  if (!projectContainer) {
+                    // Create a new tab for this project
+                    const tab = document.createElement('div');
+                    tab.className = 'project-tab';
+                    tab.dataset.project = project;
+                    tab.innerHTML = `
+                      <div class="project-tab-icon">
+                        <svg aria-hidden="true" focusable="false" data-prefix="fas" data-icon="book" class="svg-inline--fa fa-book" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512">
+                          <path fill="currentColor" d="M96 0C43 0 0 43 0 96V416c0 53 43 96 96 96H384h32c17.7 0 32-14.3 32-32s-14.3-32-32-32V384c17.7 0 32-14.3 32-32V32c0-17.7-14.3-32-32-32H384 96zm0 384H352v64H96c-17.7 0-32-14.3-32-32s14.3-32 32-32zm32-240c0-8.8 7.2-16 16-16H336c8.8 0 16 7.2 16 16s-7.2 16-16 16H144c-8.8 0-16-7.2-16-16zm16 48H336c8.8 0 16 7.2 16 16s-7.2 16-16 16H144c-8.8 0-16-7.2-16-16s7.2-16 16-16z"></path>
+                        </svg>
+                      </div>
+                      ${project} (${results.length})
+                    `;
+                    
+                    // Add tab click handler
+                    tab.addEventListener('click', function() {
+                      // Remove active class from all tabs and containers
+                      document.querySelectorAll('.project-tab').forEach(t => t.classList.remove('active'));
+                      document.querySelectorAll('.project-results').forEach(c => c.classList.remove('active'));
+                      
+                      // Add active class to this tab and its container
+                      tab.classList.add('active');
+                      const container = resultsContainer.querySelector(`.project-results[data-project="${project}"]`);
+                      if (container) container.classList.add('active');
+                    });
+                    
+                    tabsContainer.appendChild(tab);
+                    
+                    // Create a new container for this project
+                    projectContainer = document.createElement('div');
+                    projectContainer.className = 'project-results';
+                    projectContainer.dataset.project = project;
+                    
+                    // Find the load more button to insert before it (if it exists)
+                    const loadMoreBtn = resultsContainer.querySelector('.pyhc-load-more');
+                    if (loadMoreBtn) {
+                      resultsContainer.insertBefore(projectContainer, loadMoreBtn);
+                    } else {
+                      resultsContainer.appendChild(projectContainer);
+                    }
+                  } else {
+                    // If this project already exists, update the count in the tab
+                    const projectTab = tabsContainer.querySelector(`.project-tab[data-project="${project}"]`);
+                    if (projectTab) {
+                      // Get current count from tab
+                      const currentCount = parseInt(projectTab.innerText.match(/\((\d+)\)/)[1]);
+                      // Update tab text with new count
+                      const newCount = currentCount + results.length;
+                      projectTab.innerHTML = projectTab.innerHTML.replace(/\(\d+\)/, `(${newCount})`);
+                    }
+                  }
+                  
+                  // Add each result for this project to its container
+                  results.forEach(result => {
+                    const resultItem = createResultItem(result);
+                    projectContainer.appendChild(resultItem);
+                  });
+                }
+                
+                // Add pagination if needed
+                if (data.next) {
+                  // Calculate remaining pages
+                  const currentResultsCount = Array.from(document.querySelectorAll('.project-results .hit-block')).length;
+                  const totalResultsCount = data.count;
+                  const remainingResults = totalResultsCount - currentResultsCount;
+                  const remainingPages = Math.ceil(remainingResults / 50);
+                  
+                  const newLoadMoreBtn = document.createElement('button');
+                  newLoadMoreBtn.className = 'pyhc-load-more';
+                  newLoadMoreBtn.textContent = `Load more results (${remainingPages} ${remainingPages === 1 ? 'page' : 'pages'} remaining)`;
+                  newLoadMoreBtn.addEventListener('click', function() {
+                    loadNextPage(data.next);
+                  });
+                  resultsContainer.appendChild(newLoadMoreBtn);
+                }
+              })
+              .catch(retryError => {
+                // If retry fails, show error
+                if (loadMoreBtn) {
+                  loadMoreBtn.textContent = 'Error loading more results. Click to try again.';
+                  loadMoreBtn.disabled = false;
+                }
+              });
+          }, 1000);
+        } else {
+          // If no more proxies to try
+          if (loadMoreBtn) {
+            loadMoreBtn.textContent = 'Error loading more results. Click to try again.';
+            loadMoreBtn.disabled = false;
+          }
         }
       });
   }

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+sphinx==7.2.6`
	`2`	`+sphinx-rtd-theme==2.0.0`
	`3`	`+sphinx-copybutton==0.5.2`