hmm

.dockerignore

@@ -0,0 +1,9 @@
+node_modules
+npm-debug.log
+Dockerfile
+.dockerignore
+.git
+.gitignore
+README.md
+.env
+.env.*

.gitignore

@@ -0,0 +1 @@
+node_modules

Dockerfile

@@ -0,0 +1,28 @@
+# unknown
+FROM node:latest
+
+RUN apt-get update && apt-get install -y \
+    wget \
+    gnupg \
+    ca-certificates \
+    apt-transport-https \
+    chromium \
+    chromium-driver \
+    xvfb \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV CHROME_BIN=/usr/bin/chromium
+
+WORKDIR /app
+
+COPY package*.json ./
+
+RUN npm update
+RUN npm install
+# RUN npm i -g pm2
+COPY . .
+
+EXPOSE 7860
+
+#CMD ["pm2-runtime", "src/index.js"]
+CMD ["node", "src/index.js"]

LICENSE.md

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 - 2024 @zfcsoftware
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,11 @@
+---
+title: Cf
+emoji: 💻
+colorFrom: gray
+colorTo: pink
+sdk: docker
+pinned: false
+short_description: idk
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

package.json

@@ -0,0 +1,41 @@
+{
+  "name": "cf-clearance-scraper",
+  "version": "2.1.3",
+  "main": "index.js",
+  "scripts": {
+    "start": "node src/index.js",
+    "test": "node --experimental-vm-modules ./node_modules/.bin/jest --detectOpenHandles --verbose"
+  },
+  "jest": {
+    "testMatch": [
+      "**/tests/**/*.js"
+    ],
+    "verbose": true
+  },
+  "keywords": [
+    "cf-clearance",
+    "cloudflare",
+    "waf",
+    "scraper",
+    "puppeteer",
+    "xvfb",
+    "turnstile",
+    "bypass",
+    "undetected",
+    "stealth"
+  ],
+  "author": "zfcsoftware",
+  "license": "ISC",
+  "description": "This package is an experimental and educational package created for Cloudflare protections.",
+  "dependencies": {
+    "ajv": "^8.17.1",
+    "ajv-formats": "^3.0.1",
+    "body-parser": "^1.20.3",
+    "cors": "^2.8.5",
+    "dotenv": "^16.4.5",
+    "express": "^4.21.0",
+    "jest": "^29.7.0",
+    "puppeteer-real-browser": "^1.4.0",
+    "supertest": "^7.0.0"
+  }
+}

src/createBrowser.js

@@ -0,0 +1,35 @@
+const { connect } = require("puppeteer-real-browser")
+async function createBrowser() {
+    try {
+        if (global.finished == true) return
+
+        global.browser = null
+
+        // console.log('Launching the browser...');
+
+        const { browser } = await connect({
+            headless: false,
+            turnstile: true,
+            connectOption: { defaultViewport: null },
+            disableXvfb: false,
+        })
+
+        // console.log('Browser launched');
+
+        global.browser = browser;
+
+        browser.on('disconnected', async () => {
+            if (global.finished == true) return
+            console.log('Browser disconnected');
+            await new Promise(resolve => setTimeout(resolve, 3000));
+            await createBrowser();
+        })
+
+    } catch (e) {
+        console.log(e.message);
+        if (global.finished == true) return
+        await new Promise(resolve => setTimeout(resolve, 3000));
+        await createBrowser();
+    }
+}
+createBrowser()

src/fakePage.html

@@ -0,0 +1,30 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title></title>
+</head>
+
+<body>
+    <div class="turnstile"></div>
+    <script src="https://challenges.cloudflare.com/turnstile/v0/api.js?onload=onloadTurnstileCallback" defer></script>
+    <script>
+        window.onloadTurnstileCallback = function () {
+            turnstile.render('.turnstile', {
+                sitekey: '<site-key>',
+                callback: function (token) {
+                    var c = document.createElement('input');
+                    c.type = 'hidden';
+                    c.name = 'cf-response';
+                    c.value = token;
+                    document.body.appendChild(c);
+                },
+            });
+        };
+
+    </script>
+</body>
+
+</html>

src/getSource.js

@@ -0,0 +1,60 @@
+function getSource({ url, proxy }) {
+  return new Promise(async (resolve, reject) => {
+    if (!url) return reject("Missing url parameter");
+    const context = await global.browser
+      .createBrowserContext({
+        proxyServer: proxy ? `http://${proxy.host}:${proxy.port}` : undefined, // https://pptr.dev/api/puppeteer.browsercontextoptions
+      })
+      .catch(() => null);
+    if (!context) return reject("Failed to create browser context");
+
+    let isResolved = false;
+
+    var cl = setTimeout(async () => {
+      if (!isResolved) {
+        await context.close();
+        reject("Timeout Error");
+      }
+    }, global.timeOut || 60000);
+
+    try {
+      const page = await context.newPage();
+
+      if (proxy?.username && proxy?.password)
+        await page.authenticate({
+          username: proxy.username,
+          password: proxy.password,
+        });
+
+      await page.setRequestInterception(true);
+      page.on("request", async (request) => request.continue());
+      page.on("response", async (res) => {
+        try {
+          if (
+            [200, 302].includes(res.status()) &&
+            [url, url + "/"].includes(res.url())
+          ) {
+            await page
+              .waitForNavigation({ waitUntil: "load", timeout: 5000 })
+              .catch(() => {});
+            const html = await page.content();
+            await context.close();
+            isResolved = true;
+            clearInterval(cl);
+            resolve(html);
+          }
+        } catch (e) {}
+      });
+      await page.goto(url, {
+        waitUntil: "domcontentloaded",
+      });
+    } catch (e) {
+      if (!isResolved) {
+        await context.close();
+        clearInterval(cl);
+        reject(e.message);
+      }
+    }
+  });
+}
+module.exports = getSource;

src/index.js

@@ -0,0 +1,73 @@
+const express = require('express')
+const app = express()
+const port = process.env.PORT || 7860
+const bodyParser = require('body-parser')
+const authToken = process.env.authToken || null
+const cors = require('cors')
+const reqValidate = require('./reqValidate')
+
+global.browserLength = 0
+global.browserLimit = Number(process.env.browserLimit) || 20
+global.timeOut = Number(process.env.timeOut || 60000)
+
+app.use(bodyParser.json({}))
+app.use(bodyParser.urlencoded({ extended: true }))
+app.use(cors())
+if (process.env.NODE_ENV !== 'development') {
+    let server = app.listen(port, () => { console.log(`Server running on port ${port}`) })
+    try {
+        server.timeout = global.timeOut
+    } catch (e) { }
+}
+if (process.env.SKIP_LAUNCH != 'true') require('./createBrowser')
+
+const getSource = require('./getSource')
+const solveTurnstileMin = require('./solveTurnstile.min')
+const solveTurnstileMax = require('./solveTurnstile.max')
+const wafSession = require('./wafSession')
+
+app.get("/", (req, res) => {
+  res.send({ msg: "Hello World" })
+})
+
+app.post('/action', async (req, res) => {
+
+    const data = req.body
+
+    const check = reqValidate(data)
+
+    if (check !== true) return res.status(400).json({ code: 400, message: 'Bad Request', schema: check })
+
+    if (authToken && data.authToken !== authToken) return res.status(401).json({ code: 401, message: 'Unauthorized' })
+
+    if (global.browserLength >= global.browserLimit) return res.status(429).json({ code: 429, message: 'Too Many Requests' })
+
+    if (process.env.SKIP_LAUNCH != 'true' && !global.browser) return res.status(500).json({ code: 500, message: 'The scanner is not ready yet. Please try again a little later.' })
+
+    var result = { code: 500 }
+
+    global.browserLength++
+
+    switch (data.mode) {
+        case "source":
+            result = await getSource(data).then(res => { return { source: res, code: 200 } }).catch(err => { return { code: 500, message: err.message } })
+            break;
+        case "turnstile-min":
+            result = await solveTurnstileMin(data).then(res => { return { token: res, code: 200 } }).catch(err => { return { code: 500, message: err.message } })
+            break;
+        case "turnstile-max":
+            result = await solveTurnstileMax(data).then(res => { return { token: res, code: 200 } }).catch(err => { return { code: 500, message: err.message } })
+            break;
+        case "waf-session":
+            result = await wafSession(data).then(res => { return { ...res, code: 200 } }).catch(err => { return { code: 500, message: err.message } })
+            break;
+    }
+
+    global.browserLength--
+
+    res.status(result.code ?? 500).send(result)
+})
+
+app.use((req, res) => { res.status(404).json({ code: 404, message: 'Not Found' }) })
+
+if (process.env.NODE_ENV == 'development') module.exports = app

src/reqValidate.js

@@ -0,0 +1,58 @@
+const Ajv = require("ajv")
+const addFormats = require("ajv-formats")
+
+const ajv = new Ajv()
+addFormats(ajv)
+
+const schema = {
+    "type": "object",
+    "properties": {
+        "mode": {
+            "type": "string",
+            "enum": ["source", "turnstile-min", "turnstile-max", "waf-session"],
+        },
+        "proxy": {
+            "type": "object",
+            "properties": {
+                "host": { "type": "string" },
+                "port": { "type": "integer" },
+                "username": { "type": "string" },
+                "password": { "type": "string" }
+            },
+            "additionalProperties": false
+        },
+        "url": {
+            "type": "string",
+            "format": "uri",
+        },
+        "authToken": {
+            "type": "string"
+        },
+        "siteKey": {
+            "type": "string"
+        }
+    },
+    "required": ["mode", "url"],
+    "additionalProperties": false
+}
+
+// const data = {
+//     mode: "source",
+//     url: "https://example.com",
+//     proxy: {
+//         host: "localhost",
+//         port: 8080,
+//         username: "test",
+//         password: "test"
+//     },
+//     authToken: "123456"
+// }
+
+
+function validate(data) {
+    const valid = ajv.validate(schema, data)
+    if (!valid) return ajv.errors
+    else return true
+}
+
+module.exports = validate

src/solveTurnstile.max.js

@@ -0,0 +1,80 @@
+const fs = require("fs");
+function solveTurnstileMin({ url, proxy }) {
+  return new Promise(async (resolve, reject) => {
+    if (!url) return reject("Missing url parameter");
+
+    const context = await global.browser
+      .createBrowserContext({
+        proxyServer: proxy ? `http://${proxy.host}:${proxy.port}` : undefined, // https://pptr.dev/api/puppeteer.browsercontextoptions
+      })
+      .catch(() => null);
+
+    if (!context) return reject("Failed to create browser context");
+
+    let isResolved = false;
+
+    var cl = setTimeout(async () => {
+      if (!isResolved) {
+        await context.close();
+        reject("Timeout Error");
+      }
+    }, global.timeOut || 60000);
+
+    try {
+      const page = await context.newPage();
+
+      if (proxy?.username && proxy?.password)
+        await page.authenticate({
+          username: proxy.username,
+          password: proxy.password,
+        });
+        
+      await page.evaluateOnNewDocument(() => {
+        let token = null;
+        async function waitForToken() {
+          while (!token) {
+            try {
+              token = window.turnstile.getResponse();
+            } catch (e) {}
+            await new Promise((resolve) => setTimeout(resolve, 500));
+          }
+          var c = document.createElement("input");
+          c.type = "hidden";
+          c.name = "cf-response";
+          c.value = token;
+          document.body.appendChild(c);
+        }
+        waitForToken();
+      });
+
+      await page.goto(url, {
+        waitUntil: "domcontentloaded",
+      });
+
+      await page.waitForSelector('[name="cf-response"]', {
+        timeout: 60000,
+      });
+      const token = await page.evaluate(() => {
+        try {
+          return document.querySelector('[name="cf-response"]').value;
+        } catch (e) {
+          return null;
+        }
+      });
+      isResolved = true;
+      clearInterval(cl);
+      await context.close();
+      if (!token || token.length < 10) return reject("Failed to get token");
+      return resolve(token);
+    } catch (e) {
+      console.log(e);
+
+      if (!isResolved) {
+        await context.close();
+        clearInterval(cl);
+        reject(e.message);
+      }
+    }
+  });
+}
+module.exports = solveTurnstileMin;

src/solveTurnstile.min.js

@@ -0,0 +1,80 @@
+function solveTurnstileMin({ url, proxy, siteKey }) {
+  return new Promise(async (resolve, reject) => {
+    if (!url) return reject("Missing url parameter");
+    if (!siteKey) return reject("Missing siteKey parameter");
+
+    const context = await global.browser
+      .createBrowserContext({
+        proxyServer: proxy ? `http://${proxy.host}:${proxy.port}` : undefined, // https://pptr.dev/api/puppeteer.browsercontextoptions
+      })
+      .catch(() => null);
+    if (!context) return reject("Failed to create browser context");
+
+    let isResolved = false;
+
+    var cl = setTimeout(async () => {
+      if (!isResolved) {
+        await context.close();
+        reject("Timeout Error");
+      }
+    }, global.timeOut || 60000);
+
+    try {
+      const page = await context.newPage();
+
+      if (proxy?.username && proxy?.password)
+        await page.authenticate({
+          username: proxy.username,
+          password: proxy.password,
+        });
+
+      await page.setRequestInterception(true);
+
+      page.on("request", async (request) => {
+        if (
+          [url, url + "/"].includes(request.url()) &&
+          request.resourceType() === "document"
+        ) {
+          const response = await request.respond({
+            status: 200,
+            contentType: "text/html",
+            body: String(
+              require("fs").readFileSync("./src/fakePage.html")
+            ).replace(/<site-key>/g, siteKey),
+          });
+        } else {
+          await request.continue();
+        }
+      });
+
+      await page.goto(url, {
+        waitUntil: "domcontentloaded",
+      });
+
+      await page.waitForSelector('[name="cf-response"]', {
+        timeout: 60000,
+      });
+
+      const token = await page.evaluate(() => {
+        try {
+          return document.querySelector('[name="cf-response"]').value;
+        } catch (e) {
+          return null;
+        }
+      });
+
+      isResolved = true;
+      clearInterval(cl);
+      await context.close();
+      if (!token || token.length < 10) return reject("Failed to get token");
+      return resolve(token);
+    } catch (e) {
+      if (!isResolved) {
+        await context.close();
+        clearInterval(cl);
+        reject(e.message);
+      }
+    }
+  });
+}
+module.exports = solveTurnstileMin;

src/wafSession.js

@@ -0,0 +1,80 @@
+async function findAcceptLanguage(page) {
+  return await page.evaluate(async () => {
+    const result = await fetch("https://httpbin.org/get")
+      .then((res) => res.json())
+      .then(
+        (res) =>
+          res.headers["Accept-Language"] || res.headers["accept-language"]
+      )
+      .catch(() => null);
+    return result;
+  });
+}
+
+function getSource({ url, proxy }) {
+  return new Promise(async (resolve, reject) => {
+    if (!url) return reject("Missing url parameter");
+    const context = await global.browser
+      .createBrowserContext({
+        proxyServer: proxy ? `http://${proxy.host}:${proxy.port}` : undefined, // https://pptr.dev/api/puppeteer.browsercontextoptions
+      })
+      .catch(() => null);
+    if (!context) return reject("Failed to create browser context");
+
+    let isResolved = false;
+
+    var cl = setTimeout(async () => {
+      if (!isResolved) {
+        await context.close();
+        reject("Timeout Error");
+      }
+    }, global.timeOut || 60000);
+
+    try {
+      const page = await context.newPage();
+
+      if (proxy?.username && proxy?.password)
+        await page.authenticate({
+          username: proxy.username,
+          password: proxy.password,
+        });
+      let acceptLanguage = await findAcceptLanguage(page);
+      await page.setRequestInterception(true);
+      page.on("request", async (request) => request.continue());
+      page.on("response", async (res) => {
+        try {
+          if (
+            [200, 302].includes(res.status()) &&
+            [url, url + "/"].includes(res.url())
+          ) {
+            await page
+              .waitForNavigation({ waitUntil: "load", timeout: 5000 })
+              .catch(() => {});
+            const cookies = await page.cookies();
+            let headers = await res.request().headers();
+            delete headers["content-type"];
+            delete headers["accept-encoding"];
+            delete headers["accept"];
+            delete headers["content-length"];
+            headers["accept-language"] = acceptLanguage;
+            await context.close();
+            isResolved = true;
+            clearInterval(cl);
+            resolve({ cookies, headers });
+          }
+        } catch (e) {}
+      });
+
+      await page.goto(url, {
+        waitUntil: "domcontentloaded",
+      });
+    } catch (e) {
+      if (!isResolved) {
+        await context.close();
+        clearInterval(cl);
+        reject(e.message);
+      }
+    }
+  });
+}
+module.exports = getSource;