Google Scholar response cleanup

src/content-scripts/ddg_search.ts

@@ -98,11 +98,14 @@ export async function webSearch(search: SearchRequest, numResults: number): Prom
     if (response.url === `${BASE_URL}/lite/`) {
         results = htmlToSearchResults(response.html, numResults)
     } else {
-        const result = await Browser.runtime.sendMessage({
+        let result = await Browser.runtime.sendMessage({
             type: "get_webpage_text",
             url: response.url,
             html: response.html
         })
+        if (result.title && result.title === "Google Scholar") {
+           result = formatGoogleScholarResponse(result);
+        }
 
         return [{
             title: result.title,
@@ -113,3 +116,36 @@ export async function webSearch(search: SearchRequest, numResults: number): Prom
 
     return results
 }
+
+function formatGoogleScholarResponse(result: any): any {
+    result.body = cleanResponseText(result.body);
+    return result;
+}
+
+function cleanResponseText(text: string): string {
+    const lines = text.split('\n');
+    const cleanedLines: string[] = [];
+
+    for (const line of lines) {
+        const cleanedLine = line
+            .replace(/\[.*?\]/g, '') // Remove tags like [PDF], [HTML], etc.
+            .replace(/https?:\/\/[^\s]+/g, ' ') // Remove URLs
+            .replace(/Cite\s+/g, ' ') // Remove Cite button links
+            .replace(/Cited by \d+?/g, ' ') // Remove citation counts
+            .replace(/Related articles/g, ' ') // Remove 'Related articles'
+            .replace(/All \d+? versions/g, ' ') // Remove version counts
+            .replace(/View as HTML/g, ' ') // Remove 'View as HTML'
+            .replace(/Fulltext via \w+/g, ' ') // Remove 'Fulltext via X'
+            .replace(/Cached/g, '') // Remove 'Cached'
+            .replace(/...Save\s+/g, ' ') // Remove Save button artifact
+            .replace(/\S+\.(com|org|net|uk)/g, ' ') // Remove right-joined url artifacts
+            .replace(/arxiv:\S+/, ' ') // Remove arxiv code
+            .replace(/\.\.\./g, '.') // Trim ellipsis
+            .replace(/\s{2,}/g, ' ') // Trim inner extra spaces
+            .trim();
+        if (cleanedLine) {
+            cleanedLines.push(cleanedLine);
+        }
+    }  
+    return cleanedLines.join('\n');
+}