diff --git a/package.json b/package.json index 5afdf999..07f246b5 100644 --- a/package.json +++ b/package.json @@ -703,7 +703,8 @@ "vscode:prepublish": "webpack --mode production", "webpack": "webpack --mode development", "watch": "webpack --mode development --watch", - "tslint": "tslint --project tsconfig.json" + "tslint": "tslint --project tsconfig.json", + "test": "tsc -p tsconfig.test.json && node --test test/*.test.js" }, "devDependencies": { "@types/aws4": "^1.5.1", @@ -734,6 +735,7 @@ "filesize": "^3.3.0", "fs-extra": "^5.0.0", "got": "^11.8.6", + "@szmarczak/http-timer": "^4.0.5", "highlight.js": "^10.4.1", "http-proxy-agent": "^2.1.0", "https-proxy-agent": "^2.2.3", @@ -741,9 +743,11 @@ "iconv-lite": "^0.4.15", "jsonc-parser": "^2.0.2", "jsonpath-plus": "^0.20.1", + "js-yaml": "^4.1.0", + "@types/js-yaml": "^4.0.5", "mime-types": "^2.1.14", "node-fetch": "^2.6.7", - "node-jws": "^0.1.4", + "jws": "^4.0.0", "open": "^10.1.0", "pretty-data": "^0.40.0", "sanitize-html": "^2.13.0", diff --git a/src/utils/httpClient.ts b/src/utils/httpClient.ts index 27fac515..f3aa65dc 100644 --- a/src/utils/httpClient.ts +++ b/src/utils/httpClient.ts @@ -1,5 +1,8 @@ import * as fs from 'fs-extra'; +import * as http from 'http'; +import * as https from 'https'; import * as iconv from 'iconv-lite'; +import fetch from 'node-fetch'; import * as path from 'path'; import { CookieJar, Store } from 'tough-cookie'; import * as url from 'url'; @@ -8,11 +11,12 @@ import { RequestHeaders, ResponseHeaders } from '../models/base'; import { IRestClientSettings, SystemSettings } from '../models/configurationSettings'; import { HttpRequest } from '../models/httpRequest'; import { HttpResponse } from '../models/httpResponse'; +import Logger from '../logger'; import { awsCognito } from './auth/awsCognito'; import { awsSignature } from './auth/awsSignature'; import { digest } from './auth/digest'; import { MimeUtility } from './mimeUtility'; -import { getHeader, removeHeader } from './misc'; +import { getHeader, hasHeader, removeHeader } from './misc'; import { convertBufferToStream, convertStreamToBuffer } from './streamUtility'; import { UserDataManager } from './userDataManager'; import { getCurrentHttpFileName, getWorkspaceRootPath } from './workspaceUtility'; @@ -43,22 +47,61 @@ export class HttpClient { const options = await this.prepareOptions(httpRequest, settings); - let bodySize = 0; - let headersSize = 0; + const sizes = { body: 0, headers: 0 }; const requestUrl = encodeUrl(httpRequest.url); - const request: CancelableRequest> = got.default(requestUrl, options); - httpRequest.setUnderlyingRequest(request); - (request as any).on('response', res => { - if (res.rawHeaders) { - headersSize += res.rawHeaders.map(h => h.length).reduce((a, b) => a + b, 0); - headersSize += (res.rawHeaders.length) / 2; - } - res.on('data', chunk => { - bodySize += chunk.length; + + const executeRequest = async (opts: OptionsOfBufferResponseBody): Promise> => { + const request: CancelableRequest> = got.default(requestUrl, opts); + httpRequest.setUnderlyingRequest(request); + (request as any).on('error', err => { + Logger.verbose('HTTP request error', { + url: requestUrl, + code: err?.code, + message: err?.message + }); + }); + (request as any).on('response', res => { + if (res.rawHeaders) { + sizes.headers += res.rawHeaders.map(h => h.length).reduce((a, b) => a + b, 0); + sizes.headers += (res.rawHeaders.length) / 2; + } + res.on('data', chunk => { + sizes.body += chunk.length; + }); + const req = (res as any).req; + Logger.verbose('HTTP response socket', { + url: requestUrl, + reusedSocket: req?.reusedSocket, + localAddress: res.socket?.localAddress, + localPort: res.socket?.localPort, + remoteAddress: res.socket?.remoteAddress, + remotePort: res.socket?.remotePort, + bytesWritten: res.socket?.bytesWritten, + bytesRead: res.socket?.bytesRead + }); }); - }); - const response = await request; + try { + return await request; + } catch (err) { + Logger.verbose('HTTP request failed', { + url: requestUrl, + code: err?.code, + message: err?.message + }); + if (HttpClient.isConnectionResetError(err) && this.canFallbackToFetch(opts)) { + Logger.verbose('HTTP request fallback', { + url: requestUrl, + reason: err?.message || err?.code, + transport: 'node-fetch' + }); + return await this.executeWithFetch(requestUrl, opts, httpRequest, settings!, sizes); + } + throw err; + } + }; + + const response = await executeRequest(options); const contentType = response.headers['content-type']; let encoding: string | undefined; @@ -88,8 +131,8 @@ export class HttpClient { response.httpVersion, responseHeaders, bodyString, - bodySize, - headersSize, + sizes.body, + sizes.headers, bodyBuffer, response.timings.phases, new HttpRequest( @@ -123,6 +166,15 @@ export class HttpClient { // Fix #682 Do not touch original headers in httpRequest, which may be used for retry later // Simply do a shadow copy here const clonedHeaders = Object.assign({}, httpRequest.headers); + if (!hasHeader(clonedHeaders, 'Connection')) { + clonedHeaders['Connection'] = 'close'; + } + if ((typeof requestBody === 'string' || Buffer.isBuffer(requestBody)) + && !hasHeader(clonedHeaders, 'Content-Length') + && !hasHeader(clonedHeaders, 'Transfer-Encoding')) { + const length = typeof requestBody === 'string' ? Buffer.byteLength(requestBody) : requestBody.length; + clonedHeaders['Content-Length'] = String(length); + } const options: OptionsOfBufferResponseBody = { headers: clonedHeaders as any as Headers, @@ -139,7 +191,8 @@ export class HttpClient { }, https: { rejectUnauthorized: false - } + }, + http2: false }; if (settings.timeoutInMilliseconds > 0) { @@ -183,27 +236,231 @@ export class HttpClient { const certificate = this.getRequestCertificate(httpRequest.url, settings); Object.assign(options, certificate); + const directAgent = { + http: new http.Agent({ keepAlive: false }), + https: new https.Agent({ keepAlive: false }) + }; + let usingProxy = false; // set proxy if (settings.proxy && !HttpClient.ignoreProxy(httpRequest.url, settings.excludeHostsForProxy)) { - const proxyEndpoint = url.parse(settings.proxy); - if (/^https?:$/.test(proxyEndpoint.protocol || '')) { - const proxyOptions = { - host: proxyEndpoint.hostname, - port: Number(proxyEndpoint.port), - rejectUnauthorized: settings.proxyStrictSSL - }; - - const ctor = (httpRequest.url.startsWith('http:') - ? await import('http-proxy-agent') - : await import('https-proxy-agent')).default; - - options.agent = new ctor(proxyOptions); + let proxyUrl = settings.proxy.trim(); + if (!proxyUrl.includes('://')) { + proxyUrl = `http://${proxyUrl}`; + } + const proxyEndpoint = url.parse(proxyUrl); + const proxyProtocol = (proxyEndpoint.protocol || '').toLowerCase(); + if (/^https?:$/.test(proxyProtocol)) { + const proxyPort = proxyEndpoint.port + ? Number(proxyEndpoint.port) + : (proxyProtocol === 'https:' ? 443 : 80); + if (proxyEndpoint.hostname && Number.isFinite(proxyPort)) { + const proxyOptions = { + host: proxyEndpoint.hostname, + port: proxyPort, + rejectUnauthorized: settings.proxyStrictSSL + }; + + const isHttps = httpRequest.url.startsWith('https:'); + const ctor = (isHttps + ? await import('https-proxy-agent') + : await import('http-proxy-agent')).default; + const agent = new ctor(proxyOptions); + options.agent = isHttps ? { https: agent } : { http: agent }; + usingProxy = true; + } } } + if (!usingProxy) { + options.agent = directAgent; + } else if (!hasHeader(clonedHeaders, 'Proxy-Connection')) { + clonedHeaders['Proxy-Connection'] = 'close'; + } + + Logger.verbose('HTTP request options', { + method: options.method, + url: httpRequest.url, + usingProxy, + proxyConfigured: Boolean(settings.proxy), + ignoreProxy: settings.proxy ? HttpClient.ignoreProxy(httpRequest.url, settings.excludeHostsForProxy) : true, + agent: HttpClient.describeAgent(options.agent), + headers: HttpClient.pickHeaders(options.headers as Record), + timeout: options.timeout + }); + return options; } + private static describeAgent(agent: OptionsOfBufferResponseBody['agent']): string { + if (agent === false) { + return 'disabled'; + } + if (!agent) { + return 'default'; + } + const agentAny: any = agent; + if (agentAny.http || agentAny.https) { + const httpAgent = agentAny.http; + const httpsAgent = agentAny.https; + return `http:${HttpClient.agentLabel(httpAgent)} https:${HttpClient.agentLabel(httpsAgent)}`; + } + return HttpClient.agentLabel(agentAny); + } + + private static agentLabel(agent: any): string { + if (!agent) { + return 'none'; + } + const name = agent.constructor?.name || 'Agent'; + const keepAlive = agent?.options?.keepAlive; + return `${name}(keepAlive=${keepAlive ?? 'unknown'})`; + } + + private static pickHeaders(headers: Record | undefined): Record | undefined { + if (!headers) { + return undefined; + } + const allow = ['connection', 'proxy-connection', 'content-type', 'content-length', 'user-agent', 'host']; + const picked: Record = {}; + for (const key of Object.keys(headers)) { + if (allow.includes(key.toLowerCase())) { + picked[key] = headers[key]; + } + } + return picked; + } + + private static isConnectionResetError(err: any): boolean { + if (!err) { + return false; + } + if (err.code === 'ECONNRESET') { + return true; + } + return typeof err.message === 'string' && err.message.includes('socket hang up'); + } + + private canFallbackToFetch(options: OptionsOfBufferResponseBody): boolean { + const hooks = options.hooks; + if (hooks?.afterResponse && hooks.afterResponse.length > 0) { + return false; + } + return true; + } + + private async executeWithFetch( + requestUrl: string, + options: OptionsOfBufferResponseBody, + httpRequest: HttpRequest, + settings: IRestClientSettings, + sizes: { body: number; headers: number } + ): Promise> { + if (options.hooks?.beforeRequest?.length) { + for (const hook of options.hooks.beforeRequest) { + await (hook as any)(options); + } + } + + const controller = new AbortController(); + httpRequest.setUnderlyingRequest({ cancel: () => controller.abort() } as any); + + const fetchHeaders: Record = {}; + if (options.headers) { + for (const [key, value] of Object.entries(options.headers)) { + if (value === undefined) { + continue; + } + fetchHeaders[key] = Array.isArray(value) ? value.join(', ') : String(value); + } + } + + if (options.cookieJar) { + try { + const jarCookie = await (options.cookieJar as any).getCookieString(requestUrl); + const existingCookie = getHeader(fetchHeaders as any, 'Cookie') as string | undefined; + const combined = [jarCookie, existingCookie].filter(Boolean).join('; '); + if (combined) { + fetchHeaders['Cookie'] = combined; + } + } catch (err) { + Logger.verbose('HTTP cookie jar read failed', { + url: requestUrl, + message: err?.message + }); + } + } + + const timeout = typeof options.timeout === 'number' + ? options.timeout + : (options.timeout as any)?.request; + + const agentSelector = (parsedUrl: URL) => { + const agent: any = options.agent; + if (!agent || agent === false) { + return undefined; + } + if (agent.http || agent.https) { + return parsedUrl.protocol === 'https:' ? agent.https : agent.http; + } + return agent; + }; + + const fetchResponse: any = await fetch(requestUrl, { + method: options.method as string, + headers: fetchHeaders, + body: options.body as any, + redirect: settings.followRedirect ? 'follow' : 'manual', + compress: options.decompress, + timeout, + agent: agentSelector, + signal: controller.signal + } as any); + + const bodyBuffer = await fetchResponse.buffer(); + + if (options.cookieJar) { + const rawSetCookies = fetchResponse.headers?.raw?.()['set-cookie'] || []; + for (const cookie of rawSetCookies) { + try { + await (options.cookieJar as any).setCookie(cookie, requestUrl); + } catch (err) { + Logger.verbose('HTTP cookie jar write failed', { + url: requestUrl, + message: err?.message + }); + } + } + } + + const rawHeaderMap: Record = fetchResponse.headers?.raw?.() || {}; + const rawHeaders: string[] = []; + const headers: Record = {}; + for (const [name, values] of Object.entries(rawHeaderMap)) { + headers[name] = values.join(', '); + for (const value of values) { + rawHeaders.push(name, value); + } + } + + sizes.body = bodyBuffer.length; + sizes.headers = rawHeaders.reduce((sum, item) => sum + item.length, 0) + (rawHeaders.length / 2); + + return { + statusCode: fetchResponse.status, + statusMessage: fetchResponse.statusText || '', + httpVersion: '1.1', + headers, + rawHeaders, + body: bodyBuffer, + timings: { phases: {} as any }, + request: { + options: { + headers: options.headers + } + } + } as Response; + } + private decodeEscapedUnicodeCharacters(body: string): string { return body.replace(/\\u([0-9a-fA-F]{4})/gi, (_, g) => { const char = String.fromCharCode(parseInt(g, 16)); @@ -225,12 +482,20 @@ export class HttpClient { } private static ignoreProxy(requestUrl: string, excludeHostsForProxy: string[]): Boolean { + const resolvedUrl = url.parse(requestUrl); + const hostName = resolvedUrl.hostname?.toLowerCase(); + if (!hostName) { + return true; + } + + if (HttpClient.isPrivateHost(hostName)) { + return true; + } + if (!excludeHostsForProxy || excludeHostsForProxy.length === 0) { return false; } - const resolvedUrl = url.parse(requestUrl); - const hostName = resolvedUrl.hostname?.toLowerCase(); const port = resolvedUrl.port; const excludeHostsProxyList = Array.from(new Set(excludeHostsForProxy.map(eh => eh.toLowerCase()))); @@ -253,6 +518,37 @@ export class HttpClient { return false; } + private static isPrivateHost(hostName: string): boolean { + if (hostName === 'localhost' || hostName === '::1' || hostName.endsWith('.local')) { + return true; + } + + const parts = hostName.split('.'); + if (parts.length !== 4) { + return false; + } + + const nums = parts.map(part => Number(part)); + if (nums.some(n => !Number.isInteger(n) || n < 0 || n > 255)) { + return false; + } + + const [first, second] = nums; + if (first === 10 || first === 127) { + return true; + } + + if (first === 192 && second === 168) { + return true; + } + + if (first === 172 && second >= 16 && second <= 31) { + return true; + } + + return false; + } + private resolveCertificate(absoluteOrRelativePath: string | undefined): Buffer | undefined { if (absoluteOrRelativePath === undefined) { return undefined;