diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 444a3e9603..218fca6501 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -161,11 +161,16 @@ jobs: run: npm install - name: Test + # need arm64 runners or an emulator to run tests if: matrix.arch != 'arm64' env: GIT_SSH: ${{ github.workspace }}\vendor\plink.exe run: | - powershell -command "Start-Process ${{ github.workspace }}\vendor\pageant.exe ${{ github.workspace }}\vendor\private.ppk" + $encodedKey = Get-Content -Path test\private.ppk.enc + $finalPath = Join-Path -Path $HOME -ChildPath .ssh_tests\private.ppk + mkdir ~\.ssh_tests + Set-Content -Value $([System.Convert]::FromBase64String($encodedKey)) -Path $finalPath -AsByteStream + powershell -command "Start-Process .\vendor\pageant\pageant_${{ matrix.arch }}.exe $finalPath" node utils/retry npm test # You're probably wondering why this isn't a single `run: |` step, it certainly is for *nix, diff --git a/.gitignore b/.gitignore index feb22eaa22..7d1f15f049 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,4 @@ jsconfig.json test/id_rsa test/nodegit-test-rsa +test/private.ppk diff --git a/generate/templates/templates/binding.gyp b/generate/templates/templates/binding.gyp index b5e189c32c..741d964ccc 100644 --- a/generate/templates/templates/binding.gyp +++ b/generate/templates/templates/binding.gyp @@ -155,21 +155,28 @@ "-std=c++<(cxx_version)" ], }], - ["<(is_electron) == 1 and <(electron_openssl_static) == 1", { + ["<(is_electron) == 1", { + "conditions": [ + ["<(electron_openssl_static) == 1", { + "libraries": [ + "<(electron_openssl_root)/lib/libssl.a", + "<(electron_openssl_root)/lib/libcrypto.a" + ] + }], + ["<(electron_openssl_static) != 1", { + "library_dirs": [ + "<(electron_openssl_root)/lib" + ], + "libraries": [ + "-lcrypto", + "-lssl" + ] + }] + ], "include_dirs": [ "<(electron_openssl_root)/include" ], - "libraries": [ - "<(electron_openssl_root)/lib64/libssl.a", - "<(electron_openssl_root)/lib64/libcrypto.a" - ] }], - ["<(is_electron) == 1 and <(electron_openssl_static) != 1", { - "libraries": [ - "-lcrypto", - "-lssl" - ] - }] ], }], [ diff --git a/guides/install/from-source/README.md b/guides/install/from-source/README.md index eadd8306d8..64bfce18b3 100644 --- a/guides/install/from-source/README.md +++ b/guides/install/from-source/README.md @@ -65,7 +65,7 @@ npm install nodegit --msvs_version=2013 ``` ### Electron and OpenSSL ### -A local version of OpenSSL is required when building for Electron on Windows and macOS. This is due to Electron using BoringSSL, as we are not able to link to it like we are OpenSSL in Node. Additionally, OpenSSL can be statically linked on Linux by setting the `NODEGIT_OPENSSL_STATIC_LINK` environment variable to `1`. +A local version of OpenSSL is required when building for Electron. This is due to Electron using BoringSSL, as we are not able to link to it like we are OpenSSL in Node. `acquireOpenSSL.js` will attempt to download OpenSSL prebuilts from S3. If preferred, it can also be built locally by setting the environment variable `npm_config_openssl_bin_url=skip`. On macOS, this should Just Work(tm). On Windows, things are a little trickier. diff --git a/test/README.md b/test/README.md index e96c792177..e6bdccbc4d 100644 --- a/test/README.md +++ b/test/README.md @@ -17,5 +17,20 @@ Unit tests for NodeGit. Test utilities with garbage collector, index, and repository setup, that can be used in tests. -## Note -\*.enc are encrypted in base64 and unencrypted before the test suite runs as \*. +## Keys + +Note: all files are encoded in base64 in `\*.enc` and decoded before the test suite runs. + +### encrypted_rsa + - passphrase "test-password" + - registered as deploy key on [nodegit/test](https://github.com/nodegit/test) repo named "Encrypted test key" + +### id_rsa + - registered as deploy key on [nodegit/test](https://github.com/nodegit/test) repo named "Unencrypted Test Key" + +### private.ppk + - same key as id_rsa + - ppk format is used by putty/pageant and converted/generated by puttygen + +### nodegit-test-rsa + - registered as deploy key on [nodegit/private](https://github.com/nodegit/private) repo named "Tests" diff --git a/test/index.js b/test/index.js index b138525e15..abd2fc2174 100644 --- a/test/index.js +++ b/test/index.js @@ -40,6 +40,8 @@ function unencryptKey(fileName) { .toString('ascii'); fs.writeFileSync(path.join(__dirname, fileName), asciiContents, 'utf8'); } + +unencryptKey('private.ppk'); unencryptKey('id_rsa'); unencryptKey('nodegit-test-rsa'); diff --git a/test/private.ppk.enc b/test/private.ppk.enc new file mode 100644 index 0000000000..c388d98d86 --- /dev/null +++ b/test/private.ppk.enc @@ -0,0 +1,47 @@ +UHVUVFktVXNlci1LZXktRmlsZS0zOiBzc2gtcnNhCkVuY3J5cHRpb246IG5vbmUKQ29tbWVudDog +bm9uYW1lClB1YmxpYy1MaW5lczogMTIKQUFBQUIzTnphQzF5YzJFQUFBQURBUUFCQUFBQ0FRQ3dP +Y0VONis1Vkx2VXBQSGp3WnNiMHZIUEJBRzcxUmV5YwovQzAyMkhDWENJV05mOUwxSWduR25nSWU0 +OXdxdGUvWXFyYm5xMHVqOXlrSjE4Nm1GZHdoeUtBNUR2dm5SM1FmClQ5OG12VE94bnpDTUJ1QlU3 +M002UEpDYTZVekpoeXhtTjNVTU1uanEvU25oN3hycVdkYW1YQ2ROS3hRL0QxUlkKVXEzVGdLbnBI +bGxhSEI0dnoydHZvRXpBY0dzZ014aFZ4NWdoZ1YxT1Y1NzNYNFVDRXA1UVJNNHJ2WE9KMDRDLwpn +Mzl5KzFNQnladUpveno4OHBaUFg4UFVISGJOemFWd2JCeDFVRnVJdFVGeGo3WkRGK3hubDdTTDBl +RW0vc0dxClh3akhVenJlSmgwQXpZQUNYRVYwd3czZ0xSQ1pmSDEwa3RLMVlTcTFhQ0Vld1Z3MHNu +TUl3cjNSMkFjZkR5K0QKM1dhMXd0WGxNb3lDaTFhRUhwdmd1OFVOQWhteDdJVy9pcEdEZHcyaGQ0 +OVh1ZUh2MjVwaHI4S1ROdVRnWmdMNgpXYkVQckFSM3dieHp6QXJ5dmJEbWJ4N1RSbUNRQVJxZlA5 +MTlDQmF5MmwyUGgvNDFDNFJZV3pPeUxHS1F3bzFXCnRwbk9FVnBPRTRLYmRNK3g5dzBxQndEdTVa +TnRqcGd6WlBrbVBSSzhZU2NtUkRkSDRmNU1lT1dPd3Y3YTVoTUgKZHg2Q0JheGF3bUt1VFZLL2Q3 +VU9wRmRQRnhLbjJzU0l1RzZwWUlyOHJFTHF0ZU1EUnRVa2ZYUUtBWFl4NlJRVAphYmgvSlZ3ODB2 +OFJkaEs1YVRQTld3dlZ6RFpHek9VcTBjQkV2dmdjbG5PQzFqYnE4bEFyMzJlZzQvWXkyRWJpCjV2 +MkoydVBlNlE9PQpQcml2YXRlLUxpbmVzOiAyOApBQUFDQUYvalRSU1NKK0RaNkRSRDMxcVAwZm9o +QUtzbjNUaEFqL3Jyakg2blRyd2Z1dXUvZmI0OGd5MDdsTlBTCkVGNTlHTURBUXozVGl4and4N2Uv +WVlZbHB0NExHSU56ajhYTXIzYktNeFlWSlNlbEF2bHVWR3BpMEVRRDZIc2kKTHRJSlp6TkhRYjBk +U1lZenNySnBOREFJS2kvalBNOVVmWFBDbDVabmhvWHJJSWprSnFKTlltK0pZV0FmelNONQpDQkZC +UENDUXRrcWs1V3hYYVB3WlVYcExQekZWL3dqMTBRUlJ2V0IzNE1WajAwckp0SVF6K2w5NGNDbEla +bm5uCjh3MFF0TkJ6UXhqaVhLd0tWRTNDY040WkNsMWp3cTNCeWMwNkdZN21ueFEyU1hYUEwwRGNr +S2E0Sm1MZUw1S24KY3J6UmJFSWVFZUQzdWhGelUzSTByRU5RUmg2NjlJcHJhamZSekxjRnhuQzcz +QlMxUlR5Y2ZwZFNHRk9QWFQvMgpOZDM2MlhTZXA2Vk54U3Q1YTZ3a1dnSEpoMkg4ZFBjbmlESDFE +bXJBZFA5N0FrZWkxbG1YcWJ6Q0lWOXBpZDNtCmZNQUpiSm00UmhRcFREdmV1RThOaUJnSTYyVGt3 +dWNvVzJ5M0xkU3YxMzdpSkJuSlNjMTFHOUE1YkcyeVFYRTIKYWlYUXhIeW9UMlF0VmY1WklYUmFW +THViUW5jU2dMQmpnQ2QxQ2xqQjVqbFJZTXRTYzhiRmEvVEo1b1hPVk10QwpYNHhhcVo3Z3JHS05C +cUJsRFRXblRnWk1tdndQVnVjbGU3MjgzNk9KUFhvMUxqM0sza21yeEMxdU5HMDc2My96CmJLOWl2 +QWF1SFRMMnQ0cHkra0k0NC9ZcERvR2sybnhsV01kRDNMM0pVeDNRejlMeEFBQUJBUURiVG94anFO +UHIKRk9aYlg5M3g2TjU1SVdOQ1VLdXFMMVZLNjFESGFLWGZpaXdIRHlRYjEzUnhPREk2RlNXMElI +eVpqMDh5ZjBTVApJRjlzWU1BcDcvRktRTkRJalVWcjFSNmdERWdBdytjeS9naXlqMFVscUhNc2dU +UnNnSmEvSjJQYnViRDRWMzdZClJEOHpwdmtLZjhTSjBiUWpRMGlMdGFDVWJvQVA1ZmFhWGxJS3Zl +Mnh6S3VYM09Jdk0zTyt1UTBJMDZqZGtMc2IKQUcxL2dBOXpiZmltcEx3SUJCZFVJenY3aEUyajhi +aGpPR201U0lOa3M0WWVUTlRWV2R4ZnI3Yi8wVlFPSXJDdwpEUCsrQmhjOTdkK1g3WVVVZFFIM1Bw +U1dyVm9KTnNITXB1VFpoaXdDZ0ZNTU9UWEhHVmxqRThyZ2RlU2xQc2dCCjFzUWh4clJTTUIrZkFB +QUJBUUROdGU0KzFubFlLVjZFUVl4cnlKVUFLT3RNS0pmQTVYUlRzc1hoc0V5UjAwcS8KV2N3dHJm +ZjM1dzdmQVhCVndlTnplZWl5cGV2SnNZZ1JwQXU5T01ZdHdYRUJWNUYreUlCUWtpRzE3YlNlei9z +YgpueW9pV1U2QkFYR2JkQ2pmYTRlVUVURnpiY2xqdEtsZ0FCUkdqV0Q3UEZPNldmcEFqUXBjamFR +cElUNlh6WGZ1CmZXZ3RsbkZrV3lQZFd6Skw0MldZQ3pjYVNySlNGS2Z6TkR2bVIzc25ZTmRwdWxN +WlBLZUZ2bWUyVFpqdVRSUkUKN3U4S1pGeFBqUGQrQThHYW5Cck5QaWpQY0lxNUphQ1p6TDM1ZGhp +WHBiQkMyUzJYeXJLcG1jK1hKUTgycWVPdwpkNmU5b0pWMTNQN0pTc1lhRWp2UVR5TnI2QTZuNDgy +L3FtUmR1MVIzQUFBQkFRQ1c4emFSOU9lTzFwM1hpMTRYCjkwM245Y2NjUlVwZEw4RlJQNnkxaStU +WmY3ZFJaZWt5bVg2Rm1Pam0wL25vVzNIcGVaMmJiRGlBUERadVc0ZlEKbWdaMGd6bG9kRENlSnZQ +cXZTUXNRZUhKNGMyR3NUbjFUczMzWFNUVnhVS3d3amxGdkxqbzJCQXZpQWhmd2FDdwpSbFRoaWtH +L0J0LzBtWE5tMXpwcVlucUFzWldvcmo5VVZBNjJPVzgxQ1VYMS9kVTFrY1JBaWNjbFFsak5TRFhi +CnppYlU3am91enNDM0VVRURwbUdkbVRRWHhYc0I1MVVOaHl5VmgyZ0FWcUMwMDJtSndnRVVzanBx +c2FMNk03M2EKejgzODhvRHBYYnAxQ0tMRHloOHZKcElOSDBqTk5kRDF3V29BUlFlSHMvTmJRRktR +OElOSXZQc1p6MFpHZkc3Lwp1UytKClByaXZhdGUtTUFDOiBmMjY3ZTM0MzYwOTViZDc5OWYwNzQw +NDExZmJhMDM0YzZjOWNiN2VhYzk1ZDg4NDk3ZGVlYmMxNGZjZWQ0ZDU2Cg== diff --git a/test/tests/clone.js b/test/tests/clone.js index 4f2b58d275..f256e85f51 100644 --- a/test/tests/clone.js +++ b/test/tests/clone.js @@ -3,8 +3,6 @@ var assert = require("assert"); var fse = require("fs-extra"); var local = path.join.bind(path, __dirname); var _ = require("lodash"); -const util = require("util"); -const exec = util.promisify(require("child_process").exec); const generatePathWithLength = (base, length) => { @@ -237,62 +235,6 @@ describe("Clone", function() { }); }); - if (process.platform === "win32") { - it("can clone with ssh using old agent with sha1 signing support only", - async function () { - var pageant = local("../../vendor/pageant.exe"); - var old_pageant = local("../../vendor/pageant_sha1.exe"); - var privateKey = local("../../vendor/private.ppk"); - var test = this; - var url = "git@github.com:nodegit/test.git"; - var opts = { - fetchOpts: { - callbacks: { - certificateCheck: () => 0, - credentials: function(url, userName) { - return NodeGit.Credential.sshKeyFromAgent(userName); - } - } - } - }; - - try { - await exec("taskkill /im pageant.exe /f /t"); - } catch (e) { - try { - await exec("taskkill /im pageant_sha1.exe /f /t"); - } catch(e) {} - } - try { - await exec(`powershell -command "Start-Process ${old_pageant} ${privateKey}`); - } catch (e) { - try { - await exec(`powershell -command "Start-Process ${pageant} ${privateKey}`); - } catch (e) {} - return assert.fail("Cannot run old pageant"); - } - - try { - const repo = await Clone(url, clonePath, opts); - test.repository = repo; - } catch(e) { - return assert.fail("Clone error: " + e.message); - } - - try { - await exec("taskkill /im pageant_sha1.exe /f /t"); - } catch(e) {} - - try { - await exec(`powershell -command "Start-Process ${pageant} ${privateKey}`); - } catch (e) { - return assert.fail("Cannot run pageant"); - } - - return assert.ok(test.repository instanceof Repository); - }); - } - it("can clone with ssh", function() { var test = this; var url = "git@github.com:nodegit/test.git"; diff --git a/utils/acquireOpenSSL.mjs b/utils/acquireOpenSSL.mjs index 044454026d..930b5ab33c 100644 --- a/utils/acquireOpenSSL.mjs +++ b/utils/acquireOpenSSL.mjs @@ -10,6 +10,8 @@ import { createWriteStream, promises as fs } from "fs"; import { performance } from "perf_hooks"; import { promisify } from "util"; +import { hostArch, targetArch } from "./buildFlags.js"; + const pipeline = promisify(stream.pipeline); import packageJson from '../package.json' with { type: "json" }; @@ -22,26 +24,6 @@ const extractPath = path.join(vendorPath, "openssl"); const exists = (filePath) => fs.stat(filePath).then(() => true).catch(() => false); -const convertArch = (archStr) => { - const convertedArch = { - 'ia32': 'x86', - 'x86': 'x86', - 'x64': 'x64', - 'arm64': 'arm64' - }[archStr]; - - if (!convertedArch) { - throw new Error('unsupported architecture'); - } - - return convertedArch; -} - -const hostArch = convertArch(process.arch); -const targetArch = process.env.npm_config_arch - ? convertArch(process.env.npm_config_arch) - : hostArch; - const pathsToIncludeForPackage = [ "include", "lib" ]; @@ -152,20 +134,13 @@ const buildLinux = async (buildCwd) => { const configureArgs = [ buildConfig, - // Electron(at least on centos7) imports the libcups library at runtime, which has a - // dependency on the system libssl/libcrypto which causes symbol conflicts and segfaults. - // To fix this we need to hide all the openssl symbols to prevent them from being overridden - // by the runtime linker. - // "-fvisibility=hidden", - // compile static libraries - "no-shared", - // disable ssl2, ssl3, and compression - "no-ssl2", + // disable ssl3, and compression "no-ssl3", "no-comp", // set install directory `--prefix="${extractPath}"`, - `--openssldir="${extractPath}"` + `--openssldir="${extractPath}"`, + "--libdir=lib", ]; await execPromise(`./Configure ${configureArgs.join(" ")}`, { cwd: buildCwd @@ -175,12 +150,16 @@ const buildLinux = async (buildCwd) => { // only build the libraries, not the fuzzer or apps await execPromise("make build_libs", { - cwd: buildCwd + cwd: buildCwd, + maxBuffer: 10 * 1024 * 1024 }, { pipeOutput: true }); - await execPromise("make test", { - cwd: buildCwd - }, { pipeOutput: true }); + if (hostArch === targetArch) { + await execPromise("make test", { + cwd: buildCwd, + maxBuffer: 10 * 1024 * 1024 + }, { pipeOutput: true }); + } // only install software, not the docs await execPromise("make install_sw", { @@ -341,11 +320,6 @@ const buildOpenSSLIfNecessary = async ({ return; } - if (process.platform === "linux" && process.env.NODEGIT_OPENSSL_STATIC_LINK !== "1") { - console.log(`Skipping OpenSSL build, NODEGIT_OPENSSL_STATIC_LINK !== 1`); - return; - } - await removeOpenSSLIfOudated(openSSLVersion); try { @@ -396,11 +370,6 @@ const downloadOpenSSLIfNecessary = async ({ return; } - if (process.platform === "linux" && process.env.NODEGIT_OPENSSL_STATIC_LINK !== "1") { - console.log(`Skipping OpenSSL download, NODEGIT_OPENSSL_STATIC_LINK !== 1`); - return; - } - try { await fs.stat(extractPath); console.log("Skipping OpenSSL download, dir exists"); diff --git a/utils/buildFlags.js b/utils/buildFlags.js index 3c3d9d9b21..7ea87428ba 100644 --- a/utils/buildFlags.js +++ b/utils/buildFlags.js @@ -10,7 +10,29 @@ try { isGitRepo = false; } +const convertArch = (archStr) => { + const convertedArch = { + 'ia32': 'x86', + 'x86': 'x86', + 'x64': 'x64', + 'arm64': 'arm64' + }[archStr]; + + if (!convertedArch) { + throw new Error('unsupported architecture'); + } + + return convertedArch; +} + +const hostArch = convertArch(process.arch); +const targetArch = process.env.npm_config_arch + ? convertArch(process.env.npm_config_arch) + : hostArch; + module.exports = { + hostArch, + targetArch, debugBuild: !!process.env.BUILD_DEBUG, isElectron: process.env.npm_config_runtime === "electron", isGitRepo: isGitRepo, diff --git a/utils/configureLibssh2.js b/utils/configureLibssh2.js index 95328ebdb2..95fd5d3649 100644 --- a/utils/configureLibssh2.js +++ b/utils/configureLibssh2.js @@ -2,6 +2,8 @@ var cp = require("child_process"); var fse = require("fs-extra"); var path = require("path"); +const { hostArch, targetArch } = require("./buildFlags"); + const opensslVendorDirectory = path.resolve(__dirname, "..", "vendor", "openssl"); const libssh2VendorDirectory = path.resolve(__dirname, "..", "vendor", "libssh2"); const libssh2ConfigureScript = path.join(libssh2VendorDirectory, "configure"); @@ -19,20 +21,21 @@ module.exports = function retrieveExternalDependencies() { } // Run the `configure` script on Linux + let cpArgs = ` --with-libssl-prefix=${opensslVendorDirectory}`; + + const archConfigMap = { + 'x64': 'x86_64-linux-gnu', + 'arm64': 'aarch64-linux-gnu' + }; + + cpArgs += ` --build=${archConfigMap[hostArch]}`; + cpArgs += ` --host=${archConfigMap[targetArch]}`; + return new Promise(function(resolve, reject) { - var newEnv = {}; - Object.keys(process.env).forEach(function(key) { - newEnv[key] = process.env[key]; - }); - - let cpArgs = process.env.NODEGIT_OPENSSL_STATIC_LINK === '1' - ? ` --with-libssl-prefix=${opensslVendorDirectory}` - : ''; cp.exec( `${libssh2ConfigureScript}${cpArgs}`, { - cwd: libssh2VendorDirectory, - env: newEnv + cwd: libssh2VendorDirectory }, function(err, stdout, stderr) { if (err) { diff --git a/vendor/libgit2.gyp b/vendor/libgit2.gyp index f141e4d2d9..aff29d76a2 100644 --- a/vendor/libgit2.gyp +++ b/vendor/libgit2.gyp @@ -11,7 +11,6 @@ "is_clang%": 0, "is_IBMi%": "