mirror of
https://github.com/Tony0410/readlater.git
synced 2026-05-25 06:11:40 +08:00
Compare commits
1 Commits
main
...
dependabot
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7ea146fa6a |
141
package-lock.json
generated
141
package-lock.json
generated
@@ -7,14 +7,13 @@
|
||||
"": {
|
||||
"name": "readlater",
|
||||
"version": "0.1.0",
|
||||
"hasInstallScript": true,
|
||||
"dependencies": {
|
||||
"@mozilla/readability": "^0.6.0",
|
||||
"better-sqlite3": "^12.6.2",
|
||||
"drizzle-orm": "^0.45.1",
|
||||
"jsdom": "^27.4.0",
|
||||
"lucide-react": "^0.562.0",
|
||||
"next": "16.1.3",
|
||||
"next": "16.1.5",
|
||||
"react": "19.2.3",
|
||||
"react-dom": "19.2.3",
|
||||
"uuid": "^13.0.0"
|
||||
@@ -2143,9 +2142,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@next/env": {
|
||||
"version": "16.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@next/env/-/env-16.1.3.tgz",
|
||||
"integrity": "sha512-BLP14oBOvZWXgfdJf9ao+VD8O30uE+x7PaV++QtACLX329WcRSJRO5YJ+Bcvu0Q+c/lei41TjSiFf6pXqnpbQA==",
|
||||
"version": "16.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@next/env/-/env-16.1.5.tgz",
|
||||
"integrity": "sha512-CRSCPJiSZoi4Pn69RYBDI9R7YK2g59vLexPQFXY0eyw+ILevIenCywzg+DqmlBik9zszEnw2HLFOUlLAcJbL7g==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@next/eslint-plugin-next": {
|
||||
@@ -2159,9 +2158,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@next/swc-darwin-arm64": {
|
||||
"version": "16.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.1.3.tgz",
|
||||
"integrity": "sha512-CpOD3lmig6VflihVoGxiR/l5Jkjfi4uLaOR4ziriMv0YMDoF6cclI+p5t2nstM8TmaFiY6PCTBgRWB57/+LiBA==",
|
||||
"version": "16.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.1.5.tgz",
|
||||
"integrity": "sha512-eK7Wdm3Hjy/SCL7TevlH0C9chrpeOYWx2iR7guJDaz4zEQKWcS1IMVfMb9UKBFMg1XgzcPTYPIp1Vcpukkjg6Q==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -2175,9 +2174,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@next/swc-darwin-x64": {
|
||||
"version": "16.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.1.3.tgz",
|
||||
"integrity": "sha512-aF4us2JXh0zn3hNxvL1Bx3BOuh8Lcw3p3Xnurlvca/iptrDH1BrpObwkw9WZra7L7/0qB9kjlREq3hN/4x4x+Q==",
|
||||
"version": "16.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.1.5.tgz",
|
||||
"integrity": "sha512-foQscSHD1dCuxBmGkbIr6ScAUF6pRoDZP6czajyvmXPAOFNnQUJu2Os1SGELODjKp/ULa4fulnBWoHV3XdPLfA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -2191,9 +2190,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@next/swc-linux-arm64-gnu": {
|
||||
"version": "16.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.1.3.tgz",
|
||||
"integrity": "sha512-8VRkcpcfBtYvhGgXAF7U3MBx6+G1lACM1XCo1JyaUr4KmAkTNP8Dv2wdMq7BI+jqRBw3zQE7c57+lmp7jCFfKA==",
|
||||
"version": "16.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.1.5.tgz",
|
||||
"integrity": "sha512-qNIb42o3C02ccIeSeKjacF3HXotGsxh/FMk/rSRmCzOVMtoWH88odn2uZqF8RLsSUWHcAqTgYmPD3pZ03L9ZAA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -2207,9 +2206,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@next/swc-linux-arm64-musl": {
|
||||
"version": "16.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.1.3.tgz",
|
||||
"integrity": "sha512-UbFx69E2UP7MhzogJRMFvV9KdEn4sLGPicClwgqnLht2TEi204B71HuVfps3ymGAh0c44QRAF+ZmvZZhLLmhNg==",
|
||||
"version": "16.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.1.5.tgz",
|
||||
"integrity": "sha512-U+kBxGUY1xMAzDTXmuVMfhaWUZQAwzRaHJ/I6ihtR5SbTVUEaDRiEU9YMjy1obBWpdOBuk1bcm+tsmifYSygfw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -2223,9 +2222,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@next/swc-linux-x64-gnu": {
|
||||
"version": "16.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.1.3.tgz",
|
||||
"integrity": "sha512-SzGTfTjR5e9T+sZh5zXqG/oeRQufExxBF6MssXS7HPeZFE98JDhCRZXpSyCfWrWrYrzmnw/RVhlP2AxQm+wkRQ==",
|
||||
"version": "16.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.1.5.tgz",
|
||||
"integrity": "sha512-gq2UtoCpN7Ke/7tKaU7i/1L7eFLfhMbXjNghSv0MVGF1dmuoaPeEVDvkDuO/9LVa44h5gqpWeJ4mRRznjDv7LA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -2239,9 +2238,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@next/swc-linux-x64-musl": {
|
||||
"version": "16.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.1.3.tgz",
|
||||
"integrity": "sha512-HlrDpj0v+JBIvQex1mXHq93Mht5qQmfyci+ZNwGClnAQldSfxI6h0Vupte1dSR4ueNv4q7qp5kTnmLOBIQnGow==",
|
||||
"version": "16.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.1.5.tgz",
|
||||
"integrity": "sha512-bQWSE729PbXT6mMklWLf8dotislPle2L70E9q6iwETYEOt092GDn0c+TTNj26AjmeceSsC4ndyGsK5nKqHYXjQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -2255,9 +2254,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@next/swc-win32-arm64-msvc": {
|
||||
"version": "16.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.1.3.tgz",
|
||||
"integrity": "sha512-3gFCp83/LSduZMSIa+lBREP7+5e7FxpdBoc9QrCdmp+dapmTK9I+SLpY60Z39GDmTXSZA4huGg9WwmYbr6+WRw==",
|
||||
"version": "16.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.1.5.tgz",
|
||||
"integrity": "sha512-LZli0anutkIllMtTAWZlDqdfvjWX/ch8AFK5WgkNTvaqwlouiD1oHM+WW8RXMiL0+vAkAJyAGEzPPjO+hnrSNQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -2271,9 +2270,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@next/swc-win32-x64-msvc": {
|
||||
"version": "16.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.1.3.tgz",
|
||||
"integrity": "sha512-1SZVfFT8zmMB+Oblrh5OKDvUo5mYQOkX2We6VGzpg7JUVZlqe4DYOFGKYZKTweSx1gbMixyO1jnFT4thU+nNHQ==",
|
||||
"version": "16.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.1.5.tgz",
|
||||
"integrity": "sha512-7is37HJTNQGhjPpQbkKjKEboHYQnCgpVt/4rBrrln0D9nderNxZ8ZWs8w1fAtzUx7wEyYjQ+/13myFgFj6K2Ng==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -2573,6 +2572,66 @@
|
||||
"node": ">=14.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/core": {
|
||||
"version": "1.7.1",
|
||||
"dev": true,
|
||||
"inBundle": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"@emnapi/wasi-threads": "1.1.0",
|
||||
"tslib": "^2.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/runtime": {
|
||||
"version": "1.7.1",
|
||||
"dev": true,
|
||||
"inBundle": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"tslib": "^2.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/wasi-threads": {
|
||||
"version": "1.1.0",
|
||||
"dev": true,
|
||||
"inBundle": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"tslib": "^2.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@napi-rs/wasm-runtime": {
|
||||
"version": "1.1.0",
|
||||
"dev": true,
|
||||
"inBundle": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"@emnapi/core": "^1.7.1",
|
||||
"@emnapi/runtime": "^1.7.1",
|
||||
"@tybys/wasm-util": "^0.10.1"
|
||||
}
|
||||
},
|
||||
"node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@tybys/wasm-util": {
|
||||
"version": "0.10.1",
|
||||
"dev": true,
|
||||
"inBundle": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"tslib": "^2.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/tslib": {
|
||||
"version": "2.8.1",
|
||||
"dev": true,
|
||||
"inBundle": true,
|
||||
"license": "0BSD",
|
||||
"optional": true
|
||||
},
|
||||
"node_modules/@tailwindcss/oxide-win32-arm64-msvc": {
|
||||
"version": "4.1.18",
|
||||
"resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.1.18.tgz",
|
||||
@@ -6704,12 +6763,12 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/next": {
|
||||
"version": "16.1.3",
|
||||
"resolved": "https://registry.npmjs.org/next/-/next-16.1.3.tgz",
|
||||
"integrity": "sha512-gthG3TRD+E3/mA0uDQb9lqBmx1zVosq5kIwxNN6+MRNd085GzD+9VXMPUs+GGZCbZ+GDZdODUq4Pm7CTXK6ipw==",
|
||||
"version": "16.1.5",
|
||||
"resolved": "https://registry.npmjs.org/next/-/next-16.1.5.tgz",
|
||||
"integrity": "sha512-f+wE+NSbiQgh3DSAlTaw2FwY5yGdVViAtp8TotNQj4kk4Q8Bh1sC/aL9aH+Rg1YAVn18OYXsRDT7U/079jgP7w==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@next/env": "16.1.3",
|
||||
"@next/env": "16.1.5",
|
||||
"@swc/helpers": "0.5.15",
|
||||
"baseline-browser-mapping": "^2.8.3",
|
||||
"caniuse-lite": "^1.0.30001579",
|
||||
@@ -6723,14 +6782,14 @@
|
||||
"node": ">=20.9.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@next/swc-darwin-arm64": "16.1.3",
|
||||
"@next/swc-darwin-x64": "16.1.3",
|
||||
"@next/swc-linux-arm64-gnu": "16.1.3",
|
||||
"@next/swc-linux-arm64-musl": "16.1.3",
|
||||
"@next/swc-linux-x64-gnu": "16.1.3",
|
||||
"@next/swc-linux-x64-musl": "16.1.3",
|
||||
"@next/swc-win32-arm64-msvc": "16.1.3",
|
||||
"@next/swc-win32-x64-msvc": "16.1.3",
|
||||
"@next/swc-darwin-arm64": "16.1.5",
|
||||
"@next/swc-darwin-x64": "16.1.5",
|
||||
"@next/swc-linux-arm64-gnu": "16.1.5",
|
||||
"@next/swc-linux-arm64-musl": "16.1.5",
|
||||
"@next/swc-linux-x64-gnu": "16.1.5",
|
||||
"@next/swc-linux-x64-musl": "16.1.5",
|
||||
"@next/swc-win32-arm64-msvc": "16.1.5",
|
||||
"@next/swc-win32-x64-msvc": "16.1.5",
|
||||
"sharp": "^0.34.4"
|
||||
},
|
||||
"peerDependencies": {
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
"drizzle-orm": "^0.45.1",
|
||||
"jsdom": "^27.4.0",
|
||||
"lucide-react": "^0.562.0",
|
||||
"next": "16.1.3",
|
||||
"next": "16.1.5",
|
||||
"react": "19.2.3",
|
||||
"react-dom": "19.2.3",
|
||||
"uuid": "^13.0.0"
|
||||
|
||||
@@ -36,55 +36,9 @@ export interface ExtractedArticle {
|
||||
publishedAt: Date | null;
|
||||
}
|
||||
|
||||
// Try to extract actual article URL from Google News redirect page
|
||||
function extractGoogleNewsUrl(html: string): string | null {
|
||||
// Google News embeds the real URL in various ways
|
||||
// Look for data-n-au attribute (article URL)
|
||||
const dataMatch = html.match(/data-n-au="([^"]+)"/);
|
||||
if (dataMatch) return decodeURIComponent(dataMatch[1]);
|
||||
|
||||
// Look for canonical link
|
||||
const canonicalMatch = html.match(/<link[^>]+rel=["']canonical["'][^>]+href=["']([^"']+)["']/i);
|
||||
if (canonicalMatch && !canonicalMatch[1].includes('news.google.com')) {
|
||||
return canonicalMatch[1];
|
||||
}
|
||||
|
||||
// Look for og:url that's not Google News
|
||||
const ogMatch = html.match(/<meta[^>]+property=["']og:url["'][^>]+content=["']([^"']+)["']/i);
|
||||
if (ogMatch && !ogMatch[1].includes('news.google.com')) {
|
||||
return ogMatch[1];
|
||||
}
|
||||
|
||||
// Look for article link in jsdata or similar
|
||||
const jsMatch = html.match(/jsdata="[^"]*https?:\/\/(?!news\.google\.com)[^"&\s]+/);
|
||||
if (jsMatch) {
|
||||
const urlMatch = jsMatch[0].match(/https?:\/\/[^"&\s]+/);
|
||||
if (urlMatch) return urlMatch[0];
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function extractArticle(url: string): Promise<ExtractedArticle> {
|
||||
// Resolve shortened/redirect URLs first
|
||||
let resolvedUrl = url;
|
||||
|
||||
// Follow redirects to get final URL
|
||||
try {
|
||||
const headResponse = await fetch(url, {
|
||||
method: 'HEAD',
|
||||
redirect: 'follow',
|
||||
headers: {
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
},
|
||||
});
|
||||
resolvedUrl = headResponse.url;
|
||||
} catch {
|
||||
// If HEAD fails, continue with original URL
|
||||
}
|
||||
|
||||
// Fetch the page with browser-like headers to avoid bot detection
|
||||
const response = await fetch(resolvedUrl, {
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
|
||||
@@ -122,31 +76,10 @@ export async function extractArticle(url: string): Promise<ExtractedArticle> {
|
||||
throw new Error(`Failed to fetch: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
let html = await response.text();
|
||||
let finalUrl = resolvedUrl;
|
||||
|
||||
// Check if we landed on Google News - need to extract actual article URL
|
||||
if (resolvedUrl.includes('news.google.com')) {
|
||||
const realUrl = extractGoogleNewsUrl(html);
|
||||
if (realUrl) {
|
||||
// Fetch the actual article
|
||||
const articleResponse = await fetch(realUrl, {
|
||||
headers: {
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
},
|
||||
});
|
||||
if (articleResponse.ok) {
|
||||
html = await articleResponse.text();
|
||||
finalUrl = realUrl;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const html = await response.text();
|
||||
const cleanedHtml = stripStyles(html);
|
||||
const dom = new JSDOM(cleanedHtml, {
|
||||
url: finalUrl,
|
||||
url,
|
||||
virtualConsole: createVirtualConsole(),
|
||||
});
|
||||
const document = dom.window.document;
|
||||
@@ -203,7 +136,7 @@ export async function extractArticle(url: string): Promise<ExtractedArticle> {
|
||||
return {
|
||||
title: article.title || "Untitled",
|
||||
author: article.byline || null,
|
||||
siteName: article.siteName || new URL(finalUrl).hostname,
|
||||
siteName: article.siteName || new URL(url).hostname,
|
||||
excerpt: article.excerpt || null,
|
||||
content,
|
||||
textContent,
|
||||
|
||||
Reference in New Issue
Block a user