From a356283c28767829a3626a799dabb7520747cdef Mon Sep 17 00:00:00 2001 From: Thibault Zanini Date: Fri, 28 Mar 2014 16:32:32 +0100 Subject: [PATCH 1/7] Improvements with checking git urls --- .../package-manager/package-manager-tools.js | 77 +++++++++++++++++-- 1 file changed, 72 insertions(+), 5 deletions(-) diff --git a/container/package-manager/package-manager-tools.js b/container/package-manager/package-manager-tools.js index 24a9a0a7..b0f8e26f 100644 --- a/container/package-manager/package-manager-tools.js +++ b/container/package-manager/package-manager-tools.js @@ -28,14 +28,81 @@ PackageManagerTools.isVersionValid = function (version) { }; /** - * Checks if a string is a Git url. + * Checks if a string is a GitHub url. * @function - * @param {String} url. + * @param {String} gitUrl. * @return {Boolean} */ -PackageManagerTools.isGitUrl = function (url) { +PackageManagerTools.isGitUrl = function (gitUrl) { + var response = false; + + if (typeof gitUrl === 'string') { + var result = /^(git|https?|ssh)?:\/\//.exec(gitUrl); + + if (Array.isArray(result) && result.length > 1) { + var protocol = result[1]; + + if (protocol === "https" || protocol === "http") { // http[s]:// case + response = this.isHttpGitUrl(gitUrl); + + } else if (protocol === "ssh") { // ssh:// case + response = this.isSecureShellGitUrl(gitUrl); + + } else { // git:// case + response = /^git:\/\/(?:[\w\-\.~]+@)?github\.com\/[\/\w\.\-:~\?]*\/(?:[0-9a-zA-Z~][\w\-\.~]*)\.git(?:#[\w\-\.~]*)?$/.test(gitUrl); + } + } + } + + return response; +}; + +PackageManagerTools.isSecureShellGitUrl = function (url) { + return typeof url === 'string' ? + /^ssh:\/\/[\w\-\.~]+@github\.com:[\/\w\.\-:~\?]*\/(?:[0-9a-zA-Z~][\w\-\.~]*)\.git(?:#[\w\-\.~]*)?$/.test(url) : false; +}; + +PackageManagerTools.isHttpGitUrl = function (url) { return typeof url === 'string' ? - (/^git(\+https?|\+ssh)?:\/\/([\w\-\.~]+@)?[\/\w\.\-:~\?]*\/([0-9a-zA-Z~][\w\-\.~]*)\.git(#[\w\-\.~]*)?$/).exec(url) : false; + /^https?:\/\/(?:[\w\-\.~]+@)?github\.com\/[\/\w\.\-:~\?]*\/(?:[0-9a-zA-Z~][\w\-\.~]*)\.git(?:#[\w\-\.~]*)?$/.test(url) : false; +}; + +PackageManagerTools.isNpmCompatibleGitUrl = function (gitUrl) { + var response = false; + + if (typeof gitUrl === 'string' && /^git(?:\+https?|\+ssh)?:\/\//.test(gitUrl)) { // npm compatible git url + var url = gitUrl.replace(/^git\+/, ""); + response = this.isGitUrl(url); + } + + return response; +}; + +PackageManagerTools.transformGitUrlToHttpGitUrl = function (gitUrl, secure) { + var urlTransformed = null; + + if (typeof gitUrl === 'string') { + var url = null; + + if (this.isNpmCompatibleGitUrl(gitUrl)) { + url = gitUrl.replace(/^git\+/, ""); + + } else if (this.isGitUrl(gitUrl)) { + url = gitUrl; + } + + if (url) { + var patternProtocol = (secure || typeof secure === "undefined") ? 'https://' : 'http://'; + + if (this.isSecureShellGitUrl(url)) { + url = url.replace(/@github.com:/, "@github.com/"); + } + + urlTransformed = url.replace(/^(?:https?|ssh|git):\/\/(?:.*@)?/, patternProtocol); + } + } + + return urlTransformed; }; /** @@ -45,7 +112,7 @@ PackageManagerTools.isGitUrl = function (url) { * @return {Boolean} */ PackageManagerTools.isRequestValid = function (request) { - if (this.isGitUrl(request)) { // Case: Git url + if (this.isNpmCompatibleGitUrl(request)) { // Case: Git url return true; } From ec682f3a53f50512c8d585d5254e54da9ae2ef99 Mon Sep 17 00:00:00 2001 From: Thibault Zanini Date: Fri, 28 Mar 2014 16:33:02 +0100 Subject: [PATCH 2/7] Add some specs for checking the validity of git urls --- .../package-manager-tools-spec.js | 67 ++++++++++++++++++- 1 file changed, 64 insertions(+), 3 deletions(-) diff --git a/spec/container/services/package-manager-services/package-manager-tools-spec.js b/spec/container/services/package-manager-services/package-manager-tools-spec.js index 3de5356f..ff29bf97 100644 --- a/spec/container/services/package-manager-services/package-manager-tools-spec.js +++ b/spec/container/services/package-manager-services/package-manager-tools-spec.js @@ -103,11 +103,11 @@ describe("package-tools", function () { }); it("should accept valid git urls.", function() { - expect(PackageManagerTools.isRequestValid('git://git@github.com:declarativ/palette.git')).toEqual(true); + expect(PackageManagerTools.isRequestValid('git://git@github.com/declarativ/palette.git')).toEqual(true); expect(PackageManagerTools.isRequestValid('git+ssh://git@github.com:declarativ/palette.git')).toEqual(true); expect(PackageManagerTools.isRequestValid('git+ssh://git@github.com:declarativ/.git')).toEqual(false); - expect(PackageManagerTools.isRequestValid('git+http://github.com:declarativ/palette.git')).toEqual(true); - expect(PackageManagerTools.isRequestValid('git+https://git@github.com:declarativ/palette.git#445')).toEqual(true); + expect(PackageManagerTools.isRequestValid('git+http://github.com/declarativ/palette.git')).toEqual(true); + expect(PackageManagerTools.isRequestValid('git+https://git@github.com/declarativ/palette.git#445')).toEqual(true); expect(PackageManagerTools.isRequestValid('git+ftp://git@github.com:declarativ/palette.git')).toEqual(false); expect(PackageManagerTools.isRequestValid('git://git@github.com:declarativ/palette.git#93930#')).toEqual(false); @@ -115,6 +115,67 @@ describe("package-tools", function () { }); + describe("Git urls", function () { + + it("should be able to determine if an url is a git ssh url", function() { + expect(PackageManagerTools.isSecureShellGitUrl('ssh://git@github.com:declarativ/palette.git')).toEqual(true); + expect(PackageManagerTools.isSecureShellGitUrl('ssh://git@github.com:declarativ/palette.git#93930')).toEqual(true); + expect(PackageManagerTools.isSecureShellGitUrl('ssh://github.com:declarativ/palette.git')).toEqual(false); + expect(PackageManagerTools.isSecureShellGitUrl('ssh://@github.com:declarativ/palette.git')).toEqual(false); + expect(PackageManagerTools.isSecureShellGitUrl('http://git@github.com:declarativ/palette.git')).toEqual(false); + expect(PackageManagerTools.isSecureShellGitUrl('ftp://github.com:declarativ/palette.git')).toEqual(false); + + }); + + it("should be able to determine if an url is a git http{s} url", function() { + expect(PackageManagerTools.isHttpGitUrl('ssh://git@github.com:declarativ/palette.git')).toEqual(false); + expect(PackageManagerTools.isHttpGitUrl('https://github.com/declarativ/palette.git')).toEqual(true); + expect(PackageManagerTools.isHttpGitUrl('https://git@github.com/declarativ/palette.git')).toEqual(true); + expect(PackageManagerTools.isHttpGitUrl('http://github.com/declarativ/palette.git')).toEqual(true); + + }); + + it("should be able to determine if an url is a git url", function() { + expect(PackageManagerTools.isGitUrl('ssh://git@github.com:declarativ/palette.git')).toEqual(true); + expect(PackageManagerTools.isGitUrl('https://github.com/declarativ/palette.git')).toEqual(true); + expect(PackageManagerTools.isGitUrl('http://git@github.com/declarativ/palette.git')).toEqual(true); + expect(PackageManagerTools.isGitUrl('ftp://github.com:declarativ/palette.git')).toEqual(false); + + }); + + it("should be able to determine if an url is a git url compatible with npm", function() { + expect(PackageManagerTools.isNpmCompatibleGitUrl('ssh://git@github.com:declarativ/palette.git')).toEqual(false); + expect(PackageManagerTools.isNpmCompatibleGitUrl('git+https://github.com/declarativ/palette.git')).toEqual(true); + expect(PackageManagerTools.isNpmCompatibleGitUrl('git+http://github.com/declarativ/palette.git')).toEqual(true); + expect(PackageManagerTools.isNpmCompatibleGitUrl('git+ssh://git@github.com:declarativ/palette.git')).toEqual(true); + expect(PackageManagerTools.isNpmCompatibleGitUrl('git://git@github.com/declarativ/palette.git')).toEqual(true); + expect(PackageManagerTools.isNpmCompatibleGitUrl('git+ftp://github.com:declarativ/palette.git')).toEqual(false); + + }); + + it("should be able to transform a git url to a git https url", function() { + var httpsUrl = PackageManagerTools.transformGitUrlToHttpGitUrl('ssh://git@github.com:declarativ/palette.git'); + expect(httpsUrl).toEqual("https://github.com/declarativ/palette.git"); + + httpsUrl = PackageManagerTools.transformGitUrlToHttpGitUrl('git+ssh://git@github.com:declarativ/palette.git'); + expect(httpsUrl).toEqual("https://github.com/declarativ/palette.git"); + + httpsUrl = PackageManagerTools.transformGitUrlToHttpGitUrl('https://git@github.com/declarativ/palette.git'); + expect(httpsUrl).toEqual("https://github.com/declarativ/palette.git"); + + httpsUrl = PackageManagerTools.transformGitUrlToHttpGitUrl('http://git@github.com/declarativ/palette.git'); + expect(httpsUrl).toEqual("https://github.com/declarativ/palette.git"); + + httpsUrl = PackageManagerTools.transformGitUrlToHttpGitUrl('git+http://git@github.com/declarativ/palette.git'); + expect(httpsUrl).toEqual("https://github.com/declarativ/palette.git"); + + httpsUrl = PackageManagerTools.transformGitUrlToHttpGitUrl('git+ftp://git@github.com/declarativ/palette.git'); + expect(httpsUrl).toBe(null); + + }); + + }); + describe("getting a Module Object", function () { it("from the following format: name[@version].", function() { From 87f0f58d5785654f3b6acebf0c867004d15ed9a4 Mon Sep 17 00:00:00 2001 From: Thibault Zanini Date: Fri, 28 Mar 2014 16:33:56 +0100 Subject: [PATCH 3/7] Add a way to install private repository from git hub --- .../package-manager/install-package-url.js | 99 ++++++++++++++++ container/package-manager/install-packages.js | 107 ++++++++++++++++++ container/services/package-manager-service.js | 12 +- 3 files changed, 215 insertions(+), 3 deletions(-) create mode 100644 container/package-manager/install-package-url.js create mode 100644 container/package-manager/install-packages.js diff --git a/container/package-manager/install-package-url.js b/container/package-manager/install-package-url.js new file mode 100644 index 00000000..7b3afa93 --- /dev/null +++ b/container/package-manager/install-package-url.js @@ -0,0 +1,99 @@ +var PackageManagerTools = require("./package-manager-tools"), + Git = require("../git"), + FS = require("q-io/fs"), + Path = require("path"), + URL = require("url"), + Q = require("q"); + +module.exports = function installPackageFromGitUrl (url, gitHubAccessToken, fspath) { + + var _git = new Git(FS, gitHubAccessToken, true), + _httpsUrl = PackageManagerTools.transformGitUrlToHttpGitUrl(url), + _nodeModulesPath = Path.join(fspath, "node_modules"), + _packageName = null, + _pathToPackage = null; + + function _install () { + if (_httpsUrl) { + return FS.exists(_nodeModulesPath).then(function (exists) { + if (exists) { + return _installPackage(); + } + + return FS.makeDirectory(_nodeModulesPath).then(function () { + return _installPackage(); + }); + }); + } + + return Q.reject(new Error("wrong git url", url)); + } + + function _installPackage () { + var resultRegExp = /([0-9a-zA-Z~][\w\-\.~]*)\.git/.exec(_httpsUrl); + + if (Array.isArray(resultRegExp)) { + _packageName = resultRegExp[1]; + + if (_packageName) { + _pathToPackage = Path.join(_nodeModulesPath, _packageName); + + return FS.exists(_pathToPackage).then(function (exists) { + if (exists) { + return FS.removeTree(_pathToPackage).then(function () { + return _clonePackage(); + }); + } + + return _clonePackage(); + }); + } + } + + throw new Error("wrong git url", url); + } + + + function _clonePackage () { + var urlParsed = URL.parse(_httpsUrl), + commitIsh = urlParsed.hash,// tag, branch, sha + gitUrl = null; + + if (commitIsh) { + gitUrl = _httpsUrl.substring(0, _httpsUrl.length - commitIsh.length); + commitIsh = commitIsh.substr(1); // remove # symbol + } + + return _git.clone(gitUrl || _httpsUrl, _pathToPackage).then(function () { + if (commitIsh) { + return _git.command(_pathToPackage, "checkout", [commitIsh], false).then(function () { + return _cleanGitRepo(); + }); + } + + return _cleanGitRepo(); + + }).fail(function (error) { + return FS.exists(_pathToPackage).then(function (exists) { + if (exists) { + return FS.removeTree(_pathToPackage); + } + }).finally(function () { + throw new Error ("cannot install package: " + _packageName + ", error: " + error); + }); + }); + } + + function _cleanGitRepo () { + var pathToGitFolder = Path.join(_pathToPackage, ".git"); + + return FS.removeTree(pathToGitFolder).then(function () { + return { + name: _packageName, + version: url + }; + }); + } + + return _install(); +}; diff --git a/container/package-manager/install-packages.js b/container/package-manager/install-packages.js new file mode 100644 index 00000000..6294e2d5 --- /dev/null +++ b/container/package-manager/install-packages.js @@ -0,0 +1,107 @@ +var PackageManagerTools = require("./package-manager-tools"), + installPackageFromGitUrl = require("./install-package-url"), + listDependencies = require("./list-dependencies"), + Q = require("q"), + FS = require("q-io/fs"), + Path = require("path"), + execNpm = require('../package-manager/exec-npm'); + +module.exports = function installPackages (packages, gitHubAccessToken, fsPath) { + var _packagesFiltered = { + npmPackages : [], + gitPackages: [] + }, + + _packagesInstalled = [], + + _regularDependencyCategory = 'regular'; + + function _install () { + _filterPackages(); // separate packages that can be installed from a git url or with npm. + + if (_packagesFiltered.npmPackages.length > 0) { + return execNpm(execNpm.COMMANDS.INSTALL, _packagesFiltered.npmPackages, fsPath).then(function (packagesInstalled) { + _packagesInstalled = _packagesInstalled.concat(packagesInstalled); + + return _installGitPackages(); // install eventual packages from a git url. + }); + } else if (_packagesFiltered.gitPackages.length > 0) { + return _installGitPackages(); + } + + return Q.reject(new Error("no packages to install")); + } + + function _installGitPackages () { + if (_packagesFiltered.gitPackages.length > 0) { + return Q.all(_packagesFiltered.gitPackages.map(function (url) { + return installPackageFromGitUrl(url, gitHubAccessToken, fsPath).then(function (packageInstalled) { // clone package + var pathToPackage = Path.join(fsPath, "node_modules", packageInstalled.name); + + // Read package.json file from the recent cloned package, + // in order to determine if we need to install other dependencies. + return listDependencies(FS, pathToPackage, false).then(function (packageJson) { + if (packageJson.name !== packageInstalled.name) { // package name can be different from the folder name. + packageInstalled.name = packageJson.name; + } + + _packagesInstalled.push(packageInstalled); + + var packagesToInstall = _findPackagesToInstall(packageJson.children); + + if (Array.isArray(packagesToInstall) && packagesToInstall.length > 0) { + return installPackages(packagesToInstall, gitHubAccessToken, pathToPackage); + } + }); + }); + })); + } + + return void 0; + } + + function _findPackagesToInstall (dependencyListRaw) { + var dependencyList = [], + dependenciesToInstall = []; + + Object.keys(dependencyListRaw).forEach(function (key) { + var dependencyCategory = dependencyListRaw[key]; + + if (key === _regularDependencyCategory && Array.isArray(dependencyCategory)) { // install just regular dependencies + dependencyList = dependencyList.concat(dependencyCategory); + } + }); + + dependencyList.forEach(function (dependency) { + if (PackageManagerTools.isNpmCompatibleGitUrl(dependency.version)) { + dependenciesToInstall.push(dependency.version); + } else { + dependenciesToInstall.push(dependency.name); + } + }); + + return dependenciesToInstall; + } + + function _filterPackages () { + if (Array.isArray(packages)) { + packages.forEach(function (packageToInstall) { + _filterPackage(packageToInstall); + }); + } else if (typeof packages === "string") { + _filterPackage(packages); + } + } + + function _filterPackage (packageToInstall) { + if (PackageManagerTools.isNpmCompatibleGitUrl(packageToInstall)) { // git url + _packagesFiltered.gitPackages.push(packageToInstall); + } else { + _packagesFiltered.npmPackages.push(packageToInstall); // packageName[@version] + } + } + + return _install().then(function () { + return _packagesInstalled; + }); +}; diff --git a/container/services/package-manager-service.js b/container/services/package-manager-service.js index c1b0aac7..ed6963f7 100644 --- a/container/services/package-manager-service.js +++ b/container/services/package-manager-service.js @@ -2,6 +2,8 @@ var listDependencies = require('../package-manager/list-dependencies'); var FileService = require('./file-service'); var RemovePackage = require('../package-manager/remove-package'); var SearchPackages = require('../package-manager/search-packages'); +var PackageManagerTools = require('../package-manager/package-manager-tools'); +var installPackages = require('../package-manager/install-packages'); var execNpm = require('../package-manager/exec-npm'); //FIXME use fs from the service once the function “removeTree” of QFS would have be fixed after having reroot it. @@ -38,7 +40,7 @@ function PackageManagerService (session, fs, environment, pathname, fsPath) { }; service.installPackages = function (requestedPackages) { - return execNpm(execNpm.COMMANDS.INSTALL, requestedPackages, fsPath); + return installPackages(requestedPackages, session.githubAccessToken, fsPath); }; service.removePackage= function (packageName) { @@ -74,12 +76,16 @@ function PackageManagerService (session, fs, environment, pathname, fsPath) { dependenciesToInstall.forEach(function (dependency) { if (dependency.missing) { - requestedPackages.push(dependency.name); + if (PackageManagerTools.isNpmCompatibleGitUrl(dependency.version)) { + requestedPackages.push(dependency.version); + } else { + requestedPackages.push(dependency.name); + } } }); if (requestedPackages.length > 0) { - return execNpm(execNpm.COMMANDS.INSTALL, requestedPackages, fsPath); + return installPackages(requestedPackages, session.githubAccessToken, fsPath); } }); }; From 9f5425c3952b69f655feb91b2b624c106a301393 Mon Sep 17 00:00:00 2001 From: Thibault Zanini Date: Mon, 31 Mar 2014 18:36:30 +0200 Subject: [PATCH 4/7] Optimize dependencies installing Avoid to re-install a dependency that is already installed at a higher level for a dependency that could depend on it. --- container/package-manager/install-packages.js | 118 +++++++++++++++--- 1 file changed, 101 insertions(+), 17 deletions(-) diff --git a/container/package-manager/install-packages.js b/container/package-manager/install-packages.js index 6294e2d5..392fece3 100644 --- a/container/package-manager/install-packages.js +++ b/container/package-manager/install-packages.js @@ -4,32 +4,37 @@ var PackageManagerTools = require("./package-manager-tools"), Q = require("q"), FS = require("q-io/fs"), Path = require("path"), + semver = require("semver"), execNpm = require('../package-manager/exec-npm'); -module.exports = function installPackages (packages, gitHubAccessToken, fsPath) { +module.exports = function installPackages (packages, gitHubAccessToken, fsPath, dependencyTree, currentDependency) { var _packagesFiltered = { npmPackages : [], gitPackages: [] }, _packagesInstalled = [], - + _dependencyTree = dependencyTree || null, + _currentDependency = currentDependency || null, _regularDependencyCategory = 'regular'; function _install () { - _filterPackages(); // separate packages that can be installed from a git url or with npm. + return _buildDependencyTree().then(function () { + _filterPackages(); // separate packages that can be installed from a git url or with npm. - if (_packagesFiltered.npmPackages.length > 0) { - return execNpm(execNpm.COMMANDS.INSTALL, _packagesFiltered.npmPackages, fsPath).then(function (packagesInstalled) { - _packagesInstalled = _packagesInstalled.concat(packagesInstalled); + if (_packagesFiltered.npmPackages.length > 0) { + return execNpm(execNpm.COMMANDS.INSTALL, _packagesFiltered.npmPackages, fsPath).then(function (packagesInstalled) { + _packagesInstalled = _packagesInstalled.concat(packagesInstalled); + _addDependenciesToDependencyNode(packagesInstalled); - return _installGitPackages(); // install eventual packages from a git url. - }); - } else if (_packagesFiltered.gitPackages.length > 0) { - return _installGitPackages(); - } + return _installGitPackages(); // install eventual packages from a git url. + }); + } else if (_packagesFiltered.gitPackages.length > 0) { + return _installGitPackages(); + } - return Q.reject(new Error("no packages to install")); + throw new Error("no packages to install"); + }); } function _installGitPackages () { @@ -47,10 +52,11 @@ module.exports = function installPackages (packages, gitHubAccessToken, fsPath) _packagesInstalled.push(packageInstalled); - var packagesToInstall = _findPackagesToInstall(packageJson.children); + var dependencyNodeInstalled = _addDependencyToDependencyNode(packageInstalled, _currentDependency), + packagesToInstall = _findPackagesToInstall(packageJson.children, dependencyNodeInstalled); if (Array.isArray(packagesToInstall) && packagesToInstall.length > 0) { - return installPackages(packagesToInstall, gitHubAccessToken, pathToPackage); + return installPackages(packagesToInstall, gitHubAccessToken, pathToPackage, _dependencyTree, dependencyNodeInstalled); } }); }); @@ -60,7 +66,7 @@ module.exports = function installPackages (packages, gitHubAccessToken, fsPath) return void 0; } - function _findPackagesToInstall (dependencyListRaw) { + function _findPackagesToInstall (dependencyListRaw, dependencyNode) { var dependencyList = [], dependenciesToInstall = []; @@ -76,7 +82,9 @@ module.exports = function installPackages (packages, gitHubAccessToken, fsPath) if (PackageManagerTools.isNpmCompatibleGitUrl(dependency.version)) { dependenciesToInstall.push(dependency.version); } else { - dependenciesToInstall.push(dependency.name); + if (!_searchSubstituteParentNode(dependencyNode, dependency.name, dependency.version)) { + dependenciesToInstall.push(dependency.name); + } } }); @@ -97,10 +105,86 @@ module.exports = function installPackages (packages, gitHubAccessToken, fsPath) if (PackageManagerTools.isNpmCompatibleGitUrl(packageToInstall)) { // git url _packagesFiltered.gitPackages.push(packageToInstall); } else { - _packagesFiltered.npmPackages.push(packageToInstall); // packageName[@version] + var module = PackageManagerTools.getModuleFromString(packageToInstall); + + if (module && !_searchSubstituteParentNode(_currentDependency, module.name, module.version)) { + _packagesFiltered.npmPackages.push(packageToInstall); // packageName[@version] + } } } + function _buildDependencyTree () { + if (!_currentDependency && !_dependencyTree) { + return listDependencies(FS, fsPath, true).then(function (dependencyTree) { + _currentDependency = _dependencyTree = {}; + _dependencyTree.name = dependencyTree.name; + _dependencyTree.version = dependencyTree.version; + _dependencyTree.children = {}; + + Object.keys(dependencyTree.children).forEach(function (dependencyCategoryKey) { + var dependencyCategory = dependencyTree.children[dependencyCategoryKey]; + + for (var i = 0, length = dependencyCategory.length; i < length; i++) { + var dependency = dependencyCategory[i]; + + if (!dependency.missing) { + _addDependencyToDependencyNode({ + name: dependency.name, + version: dependency.versionInstalled + }, _dependencyTree); + } + } + }); + }); + } + + return Q.resolve(true); + } + + function _addDependenciesToDependencyNode (dependencies) { + if (Array.isArray(dependencies)) { + dependencies.forEach(function (dependency) { + _addDependencyToDependencyNode(dependency, _currentDependency); + }); + } + } + + function _addDependencyToDependencyNode (dependency, parentNode) { + if (parentNode && dependency) { + var dependencyNode = { + name: dependency.name, + version: dependency.version, + children: {}, + parent: parentNode + }; + + parentNode.children[dependency.name] = dependencyNode; + + return dependencyNode; + } + } + + function _searchSubstituteParentNode (dependencyNode, dependencyName, dependencyRange) { + if (dependencyNode && dependencyNode.parent) { + var children = dependencyNode.parent.children, + substituteParentNode = null; + + Object.keys(children).some(function (childKey) { + var child = children[childKey]; + + if (dependencyRange && child.name === dependencyName && semver.satisfies(child.version, dependencyRange, true)) { + substituteParentNode = child; + } + + return !!substituteParentNode; + }); + + return substituteParentNode ? substituteParentNode : _searchSubstituteParentNode(dependencyNode.parent, dependencyName, dependencyRange); + } + + return false; + } + return _install().then(function () { return _packagesInstalled; }); From 5457218dfca70d502d4126706682dbac66bc8b6c Mon Sep 17 00:00:00 2001 From: Thibault Zanini Date: Tue, 1 Apr 2014 14:34:24 +0200 Subject: [PATCH 5/7] Add support pre/post install scripts. --- container/package-manager/install-packages.js | 40 +++++++++++++++---- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/container/package-manager/install-packages.js b/container/package-manager/install-packages.js index 392fece3..4d7a1adb 100644 --- a/container/package-manager/install-packages.js +++ b/container/package-manager/install-packages.js @@ -5,6 +5,7 @@ var PackageManagerTools = require("./package-manager-tools"), FS = require("q-io/fs"), Path = require("path"), semver = require("semver"), + exec = require("child_process").exec, execNpm = require('../package-manager/exec-npm'); module.exports = function installPackages (packages, gitHubAccessToken, fsPath, dependencyTree, currentDependency) { @@ -19,7 +20,7 @@ module.exports = function installPackages (packages, gitHubAccessToken, fsPath, _regularDependencyCategory = 'regular'; function _install () { - return _buildDependencyTree().then(function () { + return Q.fapply(_buildDependencyTree).then(function () { _filterPackages(); // separate packages that can be installed from a git url or with npm. if (_packagesFiltered.npmPackages.length > 0) { @@ -53,11 +54,18 @@ module.exports = function installPackages (packages, gitHubAccessToken, fsPath, _packagesInstalled.push(packageInstalled); var dependencyNodeInstalled = _addDependencyToDependencyNode(packageInstalled, _currentDependency), - packagesToInstall = _findPackagesToInstall(packageJson.children, dependencyNodeInstalled); - - if (Array.isArray(packagesToInstall) && packagesToInstall.length > 0) { - return installPackages(packagesToInstall, gitHubAccessToken, pathToPackage, _dependencyTree, dependencyNodeInstalled); - } + packagesToInstall = _findPackagesToInstall(packageJson.children, dependencyNodeInstalled), + scripts = packageJson.fileJsonRaw.scripts; + + return Q.fcall(_execScript, scripts, 'preinstall', pathToPackage).then(function () { + if (Array.isArray(packagesToInstall) && packagesToInstall.length > 0) { + return installPackages(packagesToInstall, gitHubAccessToken, pathToPackage, _dependencyTree, dependencyNodeInstalled); + } + }).then(function () { + return Q.fcall(_execScript, scripts, 'install', pathToPackage).then(function () { + return Q.fcall(_execScript, scripts, 'postinstall', pathToPackage); + }); + }); }); }); })); @@ -66,6 +74,24 @@ module.exports = function installPackages (packages, gitHubAccessToken, fsPath, return void 0; } + function _execScript (scriptsList, scriptName, cwd) { + if (scriptsList && scriptsList.hasOwnProperty(scriptName)) { + var deferred = Q.defer(); + + exec(scriptsList[scriptName], {cwd: cwd}, function (error, stdout, stderr) { + if (error !== null) { + deferred.reject(error); + } else if (stderr) { + deferred.reject(stderr); + } else { + deferred.resolve(stdout); + } + }); + + return deferred.promise; + } + } + function _findPackagesToInstall (dependencyListRaw, dependencyNode) { var dependencyList = [], dependenciesToInstall = []; @@ -137,8 +163,6 @@ module.exports = function installPackages (packages, gitHubAccessToken, fsPath, }); }); } - - return Q.resolve(true); } function _addDependenciesToDependencyNode (dependencies) { From 1b6498b9d4116501943bb0b42a3037860eb533a3 Mon Sep 17 00:00:00 2001 From: Thibault Zanini Date: Tue, 1 Apr 2014 18:51:44 +0200 Subject: [PATCH 6/7] Improve support git ssh urls [username]@github.com/repo.git --- container/package-manager/package-manager-tools.js | 8 ++++---- .../package-manager-tools-spec.js | 3 +++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/container/package-manager/package-manager-tools.js b/container/package-manager/package-manager-tools.js index b0f8e26f..a6cd02b9 100644 --- a/container/package-manager/package-manager-tools.js +++ b/container/package-manager/package-manager-tools.js @@ -37,7 +37,7 @@ PackageManagerTools.isGitUrl = function (gitUrl) { var response = false; if (typeof gitUrl === 'string') { - var result = /^(git|https?|ssh)?:\/\//.exec(gitUrl); + var result = /^(?:(git|https?|ssh)?:\/\/|[\w\-\.~]+@)/.exec(gitUrl); if (Array.isArray(result) && result.length > 1) { var protocol = result[1]; @@ -49,7 +49,7 @@ PackageManagerTools.isGitUrl = function (gitUrl) { response = this.isSecureShellGitUrl(gitUrl); } else { // git:// case - response = /^git:\/\/(?:[\w\-\.~]+@)?github\.com\/[\/\w\.\-:~\?]*\/(?:[0-9a-zA-Z~][\w\-\.~]*)\.git(?:#[\w\-\.~]*)?$/.test(gitUrl); + response = /^(?:git:\/\/(?:[\w\-\.~]+@)?|[\w\-\.~]+@)github\.com(?:\/|:)[\/\w\.\-:~\?]*\/(?:[0-9a-zA-Z~][\w\-\.~]*)\.git(?:#[\w\-\.~]*)?$/.test(gitUrl); } } } @@ -59,7 +59,7 @@ PackageManagerTools.isGitUrl = function (gitUrl) { PackageManagerTools.isSecureShellGitUrl = function (url) { return typeof url === 'string' ? - /^ssh:\/\/[\w\-\.~]+@github\.com:[\/\w\.\-:~\?]*\/(?:[0-9a-zA-Z~][\w\-\.~]*)\.git(?:#[\w\-\.~]*)?$/.test(url) : false; + /^(?:ssh:\/\/)?[\w\-\.~]+@github\.com:[\/\w\.\-:~\?]*\/(?:[0-9a-zA-Z~][\w\-\.~]*)\.git(?:#[\w\-\.~]*)?$/.test(url) : false; }; PackageManagerTools.isHttpGitUrl = function (url) { @@ -98,7 +98,7 @@ PackageManagerTools.transformGitUrlToHttpGitUrl = function (gitUrl, secure) { url = url.replace(/@github.com:/, "@github.com/"); } - urlTransformed = url.replace(/^(?:https?|ssh|git):\/\/(?:.*@)?/, patternProtocol); + urlTransformed = url.replace(/^(?:(?:https?|ssh|git):\/\/(?:[\w\-\.~]+@)?|[\w\-\.~]+@)/, patternProtocol); } } diff --git a/spec/container/services/package-manager-services/package-manager-tools-spec.js b/spec/container/services/package-manager-services/package-manager-tools-spec.js index ff29bf97..ec1d825a 100644 --- a/spec/container/services/package-manager-services/package-manager-tools-spec.js +++ b/spec/container/services/package-manager-services/package-manager-tools-spec.js @@ -169,6 +169,9 @@ describe("package-tools", function () { httpsUrl = PackageManagerTools.transformGitUrlToHttpGitUrl('git+http://git@github.com/declarativ/palette.git'); expect(httpsUrl).toEqual("https://github.com/declarativ/palette.git"); + httpsUrl = PackageManagerTools.transformGitUrlToHttpGitUrl('git@github.com/declarativ/palette.git'); + expect(httpsUrl).toEqual("https://github.com/declarativ/palette.git"); + httpsUrl = PackageManagerTools.transformGitUrlToHttpGitUrl('git+ftp://git@github.com/declarativ/palette.git'); expect(httpsUrl).toBe(null); From 69499ef9d25b94d661fcef4c7a596d3e7966f84c Mon Sep 17 00:00:00 2001 From: Thibault Zanini Date: Wed, 2 Apr 2014 13:29:09 +0200 Subject: [PATCH 7/7] Fix issue with git urls which use the git protocol and add some specs --- .../package-manager/package-manager-tools.js | 4 +- .../package-manager-tools-spec.js | 39 ++++++++++++++++++- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/container/package-manager/package-manager-tools.js b/container/package-manager/package-manager-tools.js index a6cd02b9..a40bcaba 100644 --- a/container/package-manager/package-manager-tools.js +++ b/container/package-manager/package-manager-tools.js @@ -94,8 +94,8 @@ PackageManagerTools.transformGitUrlToHttpGitUrl = function (gitUrl, secure) { if (url) { var patternProtocol = (secure || typeof secure === "undefined") ? 'https://' : 'http://'; - if (this.isSecureShellGitUrl(url)) { - url = url.replace(/@github.com:/, "@github.com/"); + if (/github.com:/.test(url)) { + url = url.replace(/github\.com:/, "github.com/"); } urlTransformed = url.replace(/^(?:(?:https?|ssh|git):\/\/(?:[\w\-\.~]+@)?|[\w\-\.~]+@)/, patternProtocol); diff --git a/spec/container/services/package-manager-services/package-manager-tools-spec.js b/spec/container/services/package-manager-services/package-manager-tools-spec.js index ec1d825a..2b075197 100644 --- a/spec/container/services/package-manager-services/package-manager-tools-spec.js +++ b/spec/container/services/package-manager-services/package-manager-tools-spec.js @@ -154,27 +154,62 @@ describe("package-tools", function () { }); it("should be able to transform a git url to a git https url", function() { + // case : ssh://username@github.com var httpsUrl = PackageManagerTools.transformGitUrlToHttpGitUrl('ssh://git@github.com:declarativ/palette.git'); expect(httpsUrl).toEqual("https://github.com/declarativ/palette.git"); + // case : git+ssh://username@github.com httpsUrl = PackageManagerTools.transformGitUrlToHttpGitUrl('git+ssh://git@github.com:declarativ/palette.git'); expect(httpsUrl).toEqual("https://github.com/declarativ/palette.git"); + // case : https://github.com + httpsUrl = PackageManagerTools.transformGitUrlToHttpGitUrl('https://github.com/declarativ/palette.git'); + expect(httpsUrl).toEqual("https://github.com/declarativ/palette.git"); + + // case : https://username@github.com httpsUrl = PackageManagerTools.transformGitUrlToHttpGitUrl('https://git@github.com/declarativ/palette.git'); expect(httpsUrl).toEqual("https://github.com/declarativ/palette.git"); + // case : https://github.com + httpsUrl = PackageManagerTools.transformGitUrlToHttpGitUrl('https://github.com/declarativ/palette.git'); + expect(httpsUrl).toEqual("https://github.com/declarativ/palette.git"); + + // case : git+https://github.com + httpsUrl = PackageManagerTools.transformGitUrlToHttpGitUrl('git+https://github.com/declarativ/palette.git'); + expect(httpsUrl).toEqual("https://github.com/declarativ/palette.git"); + + // case : http://github.com + httpsUrl = PackageManagerTools.transformGitUrlToHttpGitUrl('http://github.com/declarativ/palette.git'); + expect(httpsUrl).toEqual("https://github.com/declarativ/palette.git"); + + // case : http://username@github.com httpsUrl = PackageManagerTools.transformGitUrlToHttpGitUrl('http://git@github.com/declarativ/palette.git'); expect(httpsUrl).toEqual("https://github.com/declarativ/palette.git"); - httpsUrl = PackageManagerTools.transformGitUrlToHttpGitUrl('git+http://git@github.com/declarativ/palette.git'); + // case : git+http://github.com + httpsUrl = PackageManagerTools.transformGitUrlToHttpGitUrl('git+http://github.com/declarativ/palette.git'); + expect(httpsUrl).toEqual("https://github.com/declarativ/palette.git"); + + // case : username@github.com + httpsUrl = PackageManagerTools.transformGitUrlToHttpGitUrl('git@github.com:declarativ/palette.git'); expect(httpsUrl).toEqual("https://github.com/declarativ/palette.git"); - httpsUrl = PackageManagerTools.transformGitUrlToHttpGitUrl('git@github.com/declarativ/palette.git'); + // case : git://username@github.com + httpsUrl = PackageManagerTools.transformGitUrlToHttpGitUrl('git://git@github.com:declarativ/palette.git'); expect(httpsUrl).toEqual("https://github.com/declarativ/palette.git"); + // case : git://github.com + httpsUrl = PackageManagerTools.transformGitUrlToHttpGitUrl('git://github.com:declarativ/palette.git'); + expect(httpsUrl).toEqual("https://github.com/declarativ/palette.git"); + + // case : git://github.com [not valid] httpsUrl = PackageManagerTools.transformGitUrlToHttpGitUrl('git+ftp://git@github.com/declarativ/palette.git'); expect(httpsUrl).toBe(null); + // case : git+https://github.com: [not valid] + httpsUrl = PackageManagerTools.transformGitUrlToHttpGitUrl('git+https://git@github.com:declarativ/palette.git'); + expect(httpsUrl).toBe(null); + }); });