Skip to content

Commit e0ed919

Browse files
netptopnetptop
committed
added cloudflare support.
1 parent dcde239 commit e0ed919

14 files changed

+544
-43
lines changed

‎.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,5 @@ yarn.lock
88
.history
99
.umirc.local.js
1010
.now
11+
bundle.js
12+
worker/

‎Proxy.js

+109-35
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,34 @@
1-
var express = require('express');
2-
var proxy = require('http-proxy-middleware');
3-
const zlib = require("zlib")
4-
const fs = require("fs")
1+
const zlib = require('zlib')
52
const parse = require('url-parse')
63
const cookiejar = require('cookiejar')
74
const iconv = require('iconv-lite')
5+
const pako = require('pako')
86
const {logSave, logGet, logClear} = require('./logger')
9-
const {CookieAccessInfo, CookieJar, Cookie} = cookiejar
7+
const ProxyMiddleware = require('http-proxy-middleware')
108

119
function countUtf8Bytes(s) {
1210
var b = 0, i = 0, c
1311
for(;c=s.charCodeAt(i++);b+=c>>11?3:c>>7?2:1);
1412
return b
1513
}
1614

15+
function uint8ArrayConcat(arrays) { // for cloudflare
16+
// sum of individual array lengths
17+
let totalLength = arrays.reduce((acc, value) => acc + value.length, 0);
18+
if (!arrays.length) return null;
19+
20+
let result = new Uint8Array(totalLength);
21+
// for each array - copy it over result
22+
// next array is copied right after the previous one
23+
let length = 0;
24+
for(let array of arrays) {
25+
result.set(array, length);
26+
length += array.length;
27+
}
28+
29+
return result;
30+
}
31+
1732
var contentTypeIsText = (headers) => {
1833
if (!headers["content-type"] ||
1934
headers["content-type"].indexOf('text/') !== -1 ||
@@ -62,7 +77,7 @@ var redirect2HomePage = function({res, httpprefix, serverName,} ) {
6277
res.status(302).send(``)
6378
}
6479

65-
let getHostFromReq = ({req, serverName}) => { //return target
80+
let getHostFromReq = ({req, serverName}) => { // return target
6681
// url: http://127.0.0.1:8011/https/www.youtube.com/xxx/xxx/...
6782
let https_prefix = '/https/'
6883
let http_prefix = '/http/'
@@ -116,7 +131,9 @@ let getHostFromReq = ({req, serverName}) => { //return target
116131
}
117132
})
118133
}
119-
if (host === '') {
134+
let localServeList = ['/index.html', '/', '/favicon.png']
135+
if (host === '' ||
136+
(host === serverName && !localServeList.includes(req.url))) {
120137
if (originalHost !== '') {
121138
httpType = originalHost.split('/')[0]
122139
host = originalHost.split('/')[1]
@@ -127,12 +144,9 @@ let getHostFromReq = ({req, serverName}) => { //return target
127144
return {host, httpType}
128145
}
129146

130-
131-
let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomainRewrite, locationReplaceMap302, regReplaceMap, siteSpecificReplace, pathReplace}) => {
132-
let stream = fs.createWriteStream("web-records.csv", {flags:'a'})
133-
let handleRespond = ({req, res, body, gbFlag}) => { // text file
134-
let myRe
135-
let {host, httpType} = getHostFromReq({req, serverName})
147+
let Proxy = ({ProxyMiddleware, blockedSites, urlModify, httpprefix, serverName, port, cookieDomainRewrite, locationReplaceMap302, regReplaceMap, siteSpecificReplace, pathReplace}) => {
148+
// let stream = fs.createWriteStream("web-records.csv", {flags:'a'})
149+
var locationMod302 = ({res, serverName, httpprefix, host, httpType}) => {
136150
let location = res.getHeaders()['location']
137151
if (res.statusCode == '301' || res.statusCode == '302' || res.statusCode == '303' ||res.statusCode == '307' || res.statusCode == '308') {
138152
location = locationReplaceMap302({location, serverName, httpprefix, host, httpType})
@@ -141,9 +155,16 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
141155
res.setHeader('location', location)
142156
} catch(e) {
143157
logSave(`error: ${e}`)
144-
return
158+
return false
145159
}
146-
// return
160+
}
161+
return true
162+
}
163+
let handleRespond = ({req, res, body, gbFlag}) => { // text file
164+
let myRe
165+
let {host, httpType} = getHostFromReq({req, serverName})
166+
if (locationMod302({res, serverName, httpprefix, host, httpType}) === false) {
167+
return
147168
}
148169
// logSave(`HandleRespond(), req.url:${req.url}, req.headers:${JSON.stringify(req.headers)}`)
149170
for(let key in regReplaceMap) {
@@ -190,15 +211,23 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
190211
}
191212
res.statusCode = '302'
192213
}
193-
logSave(`6`)
194-
body = zlib.gzipSync(body) //19ms
214+
logSave(`5 after replacment,displayed string, body.length:${body.length}`)
215+
// logSave(`5 after replacment,displayed string: ${body}`)
216+
if (process.env.cloudflare === 'true') { // in cloudflare environment
217+
let enc = new TextEncoder()
218+
body = enc.encode(body)
219+
logSave(`6 cloudflare, after encoding, uint8array, body.length:${body.length}`) // body is utf-8 uint8Array now
220+
} else { //node environment
221+
body = zlib.gzipSync(body) //body is Buffer
222+
}
223+
logSave(`7`)
195224
try {
196225
res.setHeader('content-encoding', 'gzip');
197226
logSave(`handleRespond: res.statusCode:${res.statusCode}, res.headers:${JSON.stringify(res.getHeaders())}`)
198227
if (req.headers['debugflag']==='true') {
199228
res.removeHeader('content-encoding')
200229
res.setHeader('content-type','text/plain')
201-
body=logGet()
230+
body = logGet()
202231
}
203232
res.end(body);
204233
} catch(e) {
@@ -216,7 +245,7 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
216245
return target
217246
}
218247

219-
let p = proxy({
248+
let p = ProxyMiddleware({
220249
target: `https://www.google.com`,
221250
router,
222251
/*
@@ -256,14 +285,18 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
256285
let bodyLength = 0
257286
let endFlag = false
258287
proxyRes.on('data', function(data) {
259-
// body = Buffer.concat([body, data]);
260288
if (endFlag === true) {
261289
return // don't have to push it to bodyList
262290
}
263-
bodyLength += data.length
291+
bodyLength += data.length // data is Uint8Array for cloueflare, and Buffer for node environment
264292
bodyList.push(data)
265293
if (bodyLength >= 2500000 && contentTypeIsText(proxyRes.headers) !== true) {
266-
let body = Buffer.concat(bodyList)
294+
let body
295+
if (process.env.cloudflare === 'true') { // in node environment
296+
body = uint8ArrayConcat(bodyList) // body is Uint8Array for cloueflare
297+
} else {
298+
body = Buffer.concat(bodyList) // body is Buffer for node environment
299+
}
267300
let fwdStr = req.headers['X-Forwarded-For'] || req.headers['x-forwarded-for']
268301
let contentType = proxyRes.headers['content-type']
269302
let contentLen = proxyRes.headers['content-length']
@@ -280,32 +313,58 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
280313
if (endFlag === true) {
281314
return
282315
}
283-
let body = Buffer.concat(bodyList)
316+
console.log(`on end 1, bodyList.length:${bodyList.length}`)
317+
let body
318+
if (process.env.cloudflare === 'true') {
319+
body = uint8ArrayConcat(bodyList) // body is Uint8Array
320+
if (!body) { // if body is null, end the stream.
321+
locationMod302({res, serverName, httpprefix, host, httpType})
322+
res.end(body);
323+
return
324+
}
325+
} else { // in node environment
326+
body = Buffer.concat(bodyList) // body is Buffer
327+
}
328+
console.log(`on end 2, body.length:${body.length}`)
284329
let gbFlag = false
285330
if (proxyRes.headers["content-encoding"] === 'gzip' ||
286331
proxyRes.headers["content-encoding"] === 'br') { // gzip/br encoding
287332
let gunzipped
288333
try {
289-
if (proxyRes.headers["content-encoding"] === 'br') {
290-
gunzipped = zlib.brotliDecompressSync(body)
291-
logSave(`zlib.brotli...`)
292-
} else { //gzip
293-
gunzipped = zlib.gunzipSync(body)
334+
if (process.env.cloudflare === 'true') { // in cloudflare environment
335+
console.log('cloudflare environment')
336+
gunzipped = body // in cloudflare, we have gzip
337+
} else {
338+
console.log('node environment')
339+
if (proxyRes.headers["content-encoding"] === 'br') {
340+
gunzipped = zlib.brotliDecompressSync(body)
341+
logSave(`zlib.brotli...`)
342+
} else { //gzip
343+
gunzipped = zlib.gunzipSync(body)
294344
logSave(`zlib.gunzip...`)
345+
}
295346
}
296347
} catch(e) {
297348
// res.status(404).send(`{"error": "${e}"}`)
349+
console.log(`error2:${e}`)
298350
return
299351
}
352+
console.log(`on end 3`)
300353
if (contentTypeIsText(proxyRes.headers) === true) { //gzip and text
354+
console.log(`on end 4`)
301355
if (!gunzipped) {
302356
// res.status(404).send(`{"error":"failed unzip"}`)
303357
redirect2HomePage({res, httpprefix, serverName,})
304358
return
305359
}
306-
logSave(`utf-8 text...`)
360+
logSave(`utf-8 text, gunzipped.length:${gunzipped.length}`)
307361
let originBody = gunzipped
308-
body = gunzipped.toString('utf-8');
362+
if (process.env.cloudflare === 'true') { // in cloudflare environment
363+
body = new TextDecoder().decode(gunzipped) // gunzipped.toString('utf-8');
364+
logSave(`after decode, displayed string, body.length:${body.length}`)
365+
} else {
366+
body = gunzipped.toString('utf-8')
367+
}
309368
let searchBody = body.slice(0, 1000)
310369
if (searchBody.indexOf('="text/html; charset=gb') !== -1 ||
311370
searchBody.indexOf(' charset="gb') !== -1 ||
@@ -317,15 +376,16 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
317376
let fwdStr = req.headers['X-Forwarded-For'] || req.headers['x-forwarded-for'] || ''
318377
if (proxyRes.statusCode === 200 && proxyRes.headers["content-type"] &&
319378
proxyRes.headers["content-type"].indexOf('text/html') !== -1) {
320-
saveRecord({stream, fwdStr, req, host, pktLen:body.length})
379+
// saveRecord({stream, fwdStr, req, host, pktLen:body.length})
321380
}
322381
if (proxyRes.statusCode === 200 && req.url.indexOf('/sw.js') !== -1) {
323382
// fetching sw.js
324383
res.setHeader('service-worker-allowed','/')
325384
}
326-
handleRespond({req, res, body, gbFlag})
385+
handleRespond({req, res, body, gbFlag}) // body is a displayed string
327386
} else { // gzip and non-text
328387
// console.log(`2========>${logGet()}`)
388+
console.log(`on end 5`)
329389
let fwdStr = req.headers['X-Forwarded-For'] || req.headers['x-forwarded-for']
330390
let contentType = proxyRes.headers['content-type']
331391
let contentLen = proxyRes.headers['content-length']
@@ -340,7 +400,11 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
340400
contentTypeIsText(proxyRes.headers) === true) { // text with non gzip encoding
341401
logSave(`utf-8 text...`)
342402
let originBody = body
343-
body = body.toString('utf-8');
403+
if (process.env.cloudflare === 'true') { // in cloudflare environment
404+
body = new TextDecoder().decode(body) // Uint8Array(utf-8 arrayBuffer) toString('utf-8')
405+
} else { // node environment
406+
body = body.toString('utf-8');
407+
}
344408
if (body.indexOf('="text/html; charset=gb') !== -1 ||
345409
body.indexOf(' charset="gb') !== -1 ||
346410
body.indexOf('=\'text/html; charset=gb') !== -1) {
@@ -355,16 +419,22 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
355419
let contentType = proxyRes.headers['content-type']
356420
let contentLen = proxyRes.headers['content-length']
357421
console.log(`end,route:${fwdStr}, content-type:${contentType},length:${bodyLength}, content-length:${contentLen}, ${host}`)
358-
res.end(body)
422+
if (process.env.cloudflare === 'true') { // in cloudflare environment
423+
res.end(Uint8Array.from(body))
424+
} else { // node environment
425+
res.end(body)
426+
}
359427
}
360428
})
361429
const setCookieHeaders = proxyRes.headers['set-cookie'] || []
430+
console.log(`1`)
362431
let datestr = ''
363432
if (setCookieHeaders.length > 0) {
364433
let date = new Date
365434
date.setDate(date.getDate() + 1) // 一天之后过期
366435
datestr = date.toUTCString()
367436
}
437+
console.log(`2, setCookieHeaders:${JSON.stringify(setCookieHeaders)}`)
368438
const modifiedSetCookieHeaders = setCookieHeaders
369439
.map(str => new cookiejar.Cookie(str))
370440
.map(cookie => {
@@ -378,6 +448,7 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
378448
return cookie
379449
})
380450
.map(cookie => cookie.toString())
451+
console.log(`3`)
381452
let cookie_originalHost= new cookiejar.Cookie()
382453
cookie_originalHost.name = 'ORIGINALHOST'
383454
cookie_originalHost.value = `${httpType}/${host}`
@@ -386,6 +457,7 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
386457
cookie_originalHost.path = `/`
387458
cookie_originalHost.secure = false
388459
modifiedSetCookieHeaders.push(cookie_originalHost.toString())
460+
console.log(`4`)
389461
proxyRes.headers['set-cookie'] = modifiedSetCookieHeaders
390462
Object.keys(proxyRes.headers).forEach(function (key) {
391463
if (key === 'content-security-policy' ||
@@ -469,9 +541,11 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
469541
})
470542
proxyReq.setHeader('Accept-Encoding', 'gzip')
471543
proxyReq.setHeader('referer', host)
544+
console.log(`host=${host}`)
472545
if (host.indexOf('youtube.com') !== -1) {
473-
proxyReq.setHeader('User-Agent', `Opera/7.50 (Windows XP; U)`)
546+
// proxyReq.setHeader('User-Agent', `Opera/7.50 (Windows XP; U)`)
474547
// proxyReq.setHeader('User-Agent', `Opera/9.80 (Android 4.1.2; Linux; Opera Mobi/ADR-1305251841) Presto/2.11.355 Version/12.10`)
548+
proxyReq.setHeader('User-Agent', `Mozilla/5.0 (Linux; Android 6.0.1; Lenovo-A6020l36 Build/MMB29M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.93 Mobile Safari/537.36`)
475549
}
476550
logSave(`req host:${host}, req.url:${req.url}, proxyReq.query:${proxyReq.query} proxyReq.path:${proxyReq.path}, proxyReq.url:${proxyReq.url} proxyReq headers:${JSON.stringify(proxyReq.getHeaders())}`)
477551
if(host === '' || !host) {

‎api/main.js

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,13 @@
11
const fs = require('fs')
22
const path = require('path')
3+
const ProxyMiddleware = require('http-proxy-middleware')
34
var Proxy = require('../Proxy')
45

56
let { blockedSites, urlModify, httpprefix, serverName, port, locationReplaceMap302, regReplaceMap, siteSpecificReplace, pathReplace } = require('../config')
67

78
let cookieDomainRewrite = serverName
89

9-
let proxy = Proxy({ blockedSites, urlModify, httpprefix, serverName, port, cookieDomainRewrite, locationReplaceMap302, regReplaceMap, siteSpecificReplace, pathReplace })
10+
let proxy = Proxy({ ProxyMiddleware, blockedSites, urlModify, httpprefix, serverName, port, cookieDomainRewrite, locationReplaceMap302, regReplaceMap, siteSpecificReplace, pathReplace })
1011
export default (req, res) => {
1112
const dirPath = path.join(__dirname + '/..', req.url)
1213
console.log(`x-forward-for:${req.headers['x-forwarded-for']}, req.url:${req.url}`)

‎build/worker.js

+93
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)