AI Newsletter Digest improvements: fixed QP soft line break decoding, URL extraction, and content cleaning

automations/x-no-api-bot/README.md

@@ -0,0 +1,53 @@
+# X No-API Bot (Starter)
+
+Safe starter for using X without official API.
+
+## What this does
+- `login`: saves browser session cookie/state
+- `fetch`: read-only fetch from X search timeline
+- `post`: optional posting (disabled by default)
+
+## Setup
+```bash
+cd /home/openclaw/.openclaw/workspace/automations/x-no-api-bot
+cp .env.example .env
+npm install
+npx playwright install chromium
+```
+
+## 1) Login once
+```bash
+npm run login
+```
+A browser opens. Log in to X manually, then press Enter in terminal.
+
+## 2) Read-only fetch
+```bash
+npm run fetch
+```
+Outputs JSON and saves a log under `logs/`.
+
+## 3) Optional posting (off by default)
+Edit `.env`:
+```ini
+ENABLE_POSTING=true
+```
+Then draft a post:
+```bash
+npm run post -- "Hello from no-API automation"
+```
+Publish only with explicit confirm:
+```bash
+npm run post -- "Hello from no-API automation" --confirm
+```
+
+## Cron (read-only)
+Example every 30 min:
+```cron
+*/30 * * * * cd /home/openclaw/.openclaw/workspace/automations/x-no-api-bot && /usr/bin/npm run fetch >> logs/cron.log 2>&1
+```
+
+## Notes
+- UI can change and break selectors.
+- Keep activity low-volume to reduce account risk.
+- Prefer read-only monitoring unless you explicitly need posting.

automations/x-no-api-bot/package-lock.json

@@ -0,0 +1,72 @@
+{
+  "name": "x-no-api-bot",
+  "version": "0.1.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "x-no-api-bot",
+      "version": "0.1.0",
+      "dependencies": {
+        "dotenv": "^16.4.5",
+        "playwright": "^1.52.0"
+      }
+    },
+    "node_modules/dotenv": {
+      "version": "16.6.1",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.6.1.tgz",
+      "integrity": "sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==",
+      "license": "BSD-2-Clause",
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://dotenvx.com"
+      }
+    },
+    "node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "hasInstallScript": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
+    "node_modules/playwright": {
+      "version": "1.58.2",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.2.tgz",
+      "integrity": "sha512-vA30H8Nvkq/cPBnNw4Q8TWz1EJyqgpuinBcHET0YVJVFldr8JDNiU9LaWAE1KqSkRYazuaBhTpB5ZzShOezQ6A==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright-core": "1.58.2"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "optionalDependencies": {
+        "fsevents": "2.3.2"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.58.2",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.58.2.tgz",
+      "integrity": "sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==",
+      "license": "Apache-2.0",
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    }
+  }
+}

automations/x-no-api-bot/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "x-no-api-bot",
+  "version": "0.1.0",
+  "private": true,
+  "description": "No-API X automation starter using Playwright (read-only by default)",
+  "type": "module",
+  "scripts": {
+    "login": "node src/login.js",
+    "fetch": "node src/fetch.js",
+    "post": "node src/post.js"
+  },
+  "dependencies": {
+    "dotenv": "^16.4.5",
+    "playwright": "^1.52.0"
+  }
+}

automations/x-no-api-bot/src/common.js

@@ -0,0 +1,28 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import dotenv from 'dotenv';
+
+dotenv.config();
+
+export const ROOT = process.cwd();
+export const STATE_DIR = path.join(ROOT, 'state');
+export const LOG_DIR = path.join(ROOT, 'logs');
+export const SESSION_FILE = path.join(STATE_DIR, 'x-session.json');
+
+if (!fs.existsSync(STATE_DIR)) fs.mkdirSync(STATE_DIR, { recursive: true });
+if (!fs.existsSync(LOG_DIR)) fs.mkdirSync(LOG_DIR, { recursive: true });
+
+export const ENABLE_POSTING = String(process.env.ENABLE_POSTING || 'false').toLowerCase() === 'true';
+export const X_QUERY = process.env.X_QUERY || '(AI OR OpenClaw) lang:en -is:retweet';
+export const MAX_TWEETS = Number(process.env.MAX_TWEETS || 10);
+export const USER_AGENT = process.env.USER_AGENT;
+
+export function nowStamp() {
+  return new Date().toISOString();
+}
+
+export function writeJsonLog(name, data) {
+  const file = path.join(LOG_DIR, `${name}-${Date.now()}.json`);
+  fs.writeFileSync(file, JSON.stringify(data, null, 2));
+  return file;
+}

automations/x-no-api-bot/src/fetch.js

@@ -0,0 +1,65 @@
+import fs from 'node:fs';
+import { chromium } from 'playwright';
+import {
+  SESSION_FILE,
+  USER_AGENT,
+  X_QUERY,
+  MAX_TWEETS,
+  nowStamp,
+  writeJsonLog
+} from './common.js';
+
+const hasSession = fs.existsSync(SESSION_FILE);
+if (!hasSession) {
+  console.error('No session found. Run: npm run login');
+  process.exit(1);
+}
+
+const browser = await chromium.launch({ headless: true });
+const context = await browser.newContext({
+  storageState: SESSION_FILE,
+  userAgent: USER_AGENT || undefined,
+  viewport: { width: 1366, height: 900 }
+});
+
+const page = await context.newPage();
+const url = `https://x.com/search?q=${encodeURIComponent(X_QUERY)}&src=typed_query&f=live`;
+await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 });
+
+await page.waitForTimeout(2500);
+
+const tweets = await page.evaluate((maxTweets) => {
+  const out = [];
+  const articles = Array.from(document.querySelectorAll('article'));
+
+  for (const a of articles) {
+    if (out.length >= maxTweets) break;
+
+    const textNode = a.querySelector('[data-testid="tweetText"]');
+    const userNode = a.querySelector('a[role="link"][href*="/"]');
+    const timeNode = a.querySelector('time');
+    const linkNode = a.querySelector('a[href*="/status/"]');
+
+    const text = textNode?.innerText?.trim();
+    const user = userNode?.getAttribute('href') || null;
+    const when = timeNode?.getAttribute('datetime') || null;
+    const link = linkNode ? `https://x.com${linkNode.getAttribute('href')}` : null;
+
+    if (text) out.push({ text, user, when, link });
+  }
+
+  return out;
+}, MAX_TWEETS);
+
+const payload = {
+  ts: nowStamp(),
+  query: X_QUERY,
+  count: tweets.length,
+  tweets
+};
+
+const file = writeJsonLog('x-fetch', payload);
+console.log(JSON.stringify(payload, null, 2));
+console.log(`\nSaved log: ${file}`);
+
+await browser.close();

automations/x-no-api-bot/src/login.js

@@ -0,0 +1,26 @@
+import fs from 'node:fs';
+import { chromium } from 'playwright';
+import { SESSION_FILE, USER_AGENT } from './common.js';
+
+console.log('Starting X login capture...');
+console.log('A browser will open. Log in manually, then press ENTER here to save session.');
+
+const browser = await chromium.launch({ headless: false });
+const context = await browser.newContext({
+  userAgent: USER_AGENT || undefined,
+  viewport: { width: 1366, height: 900 }
+});
+const page = await context.newPage();
+await page.goto('https://x.com/i/flow/login', { waitUntil: 'domcontentloaded' });
+
+process.stdin.resume();
+await new Promise((resolve) => {
+  process.stdout.write('\nPress ENTER after login completes in browser...\n');
+  process.stdin.once('data', () => resolve());
+});
+
+await context.storageState({ path: SESSION_FILE });
+console.log(`Saved session to ${SESSION_FILE}`);
+
+await browser.close();
+process.exit(0);

automations/x-no-api-bot/src/post.js

@@ -0,0 +1,45 @@
+import fs from 'node:fs';
+import { chromium } from 'playwright';
+import { ENABLE_POSTING, SESSION_FILE, USER_AGENT } from './common.js';
+
+const text = process.argv.slice(2).join(' ').trim();
+if (!text) {
+  console.error('Usage: npm run post -- "your post text"');
+  process.exit(1);
+}
+
+if (!ENABLE_POSTING) {
+  console.error('Posting is disabled. Set ENABLE_POSTING=true in .env to enable.');
+  process.exit(1);
+}
+
+if (!fs.existsSync(SESSION_FILE)) {
+  console.error('No session found. Run: npm run login');
+  process.exit(1);
+}
+
+const browser = await chromium.launch({ headless: true });
+const context = await browser.newContext({
+  storageState: SESSION_FILE,
+  userAgent: USER_AGENT || undefined,
+  viewport: { width: 1366, height: 900 }
+});
+
+const page = await context.newPage();
+await page.goto('https://x.com/compose/post', { waitUntil: 'domcontentloaded', timeout: 60000 });
+await page.waitForSelector('[data-testid="tweetTextarea_0"]', { timeout: 20000 });
+await page.fill('[data-testid="tweetTextarea_0"]', text);
+
+// Safety: we require explicit --confirm to actually click Post
+const confirmed = process.argv.includes('--confirm');
+if (!confirmed) {
+  console.log('Draft prepared, not posted. Re-run with --confirm to publish.');
+  await browser.close();
+  process.exit(0);
+}
+
+await page.click('[data-testid="tweetButtonInline"]');
+await page.waitForTimeout(2000);
+
+console.log('Post submitted.');
+await browser.close();