Adding zpm "ci" command to install from a lock file

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 - #1024: Added flag -export-python-deps to publish command
+- #962: Adding zpm "ci" command to install from a lock file
 
 ### Fixed
 - #996: Ensure COS commands execute in exec under a dedicated, isolated context

src/cls/IPM/General/AbstractHistory.cls

@@ -8,8 +8,8 @@ Include (%IPM.Common, %IPM.Formatting)
 Class %IPM.General.AbstractHistory Extends %Persistent [ Abstract, NoExtent ]
 {
 
-/// Action of this history record. Can be load, install, uninstall or update
-Property Action As %String(VALUELIST = ",load,install,uninstall,update") [ Required ];
+/// Action of this history record. Can be load, install, ci, uninstall or update
+Property Action As %String(VALUELIST = ",load,install,ci,uninstall,update") [ Required ];
 
 /// Name of the package being logged. This is not necessarily required, e.g. when loading a nonexistent directory.
 Property Package As %IPM.DataType.ModuleName;
@@ -66,6 +66,11 @@ ClassMethod InstallInit(Package As %IPM.DataType.ModuleName) As %IPM.General.Abs
     quit ..Init("install", Package)
 }
 
+ClassMethod CleanInstallInit(Package As %IPM.DataType.ModuleName) As %IPM.General.AbstractHistory
+{
+    quit ..Init("ci", Package)
+}
+
 ClassMethod LoadInit(Package As %IPM.DataType.ModuleName = "") As %IPM.General.AbstractHistory
 {
     // Package name may not known at this point, so use a placeholder

src/cls/IPM/General/LockFile.cls

@@ -69,12 +69,12 @@ ClassMethod CreateLockFileForModule(
         $$$ThrowOnError(lockFile.Dependencies.SetAt(dependencyVal, mod.Name))
 
         // Add the dependency's repository to the lock file
-        do AddRepositoryToLockFile(.lockFile, mod.Repository, verbose)
+        do ..AddRepositoryToLockFile(.lockFile, mod.Repository, verbose)
     }
     // Add repository for base module if not already added by a dependency
     // Skip undefined repositories as that means the module was installed via the zpm "load" command
     if (module.Repository '= "") {
-        do AddRepositoryToLockFile(.lockFile, module.Repository, verbose)
+        do ..AddRepositoryToLockFile(.lockFile, module.Repository, verbose)
     }
 
     $$$ThrowOnError(lockFile.%JSONExportToStream(.lockFileJSON, "LockFileMapping"))
@@ -116,4 +116,88 @@ ClassMethod GetRepo(repoName As %String) As %IPM.Repo.Definition [ Internal ]
     }
 }
 
+ClassMethod InstallFromLockFile(
+	directory As %String,
+	ByRef params)
+{
+    set verbose = $get(params("Verbose"), 0)
+
+    set lockFilePath = ##class(%File).NormalizeFilename("module-lock.json", directory)
+    set lockFileJSON = ##class(%DynamicObject).%FromJSONFile(lockFilePath)
+
+    set repositories = lockFileJSON.%Get("repositories", {})
+    set dependencies = lockFileJSON.%Get("dependencies", {})
+
+    // Install repositories (if they don't already exist)
+    set repoIter = repositories.%GetIterator()
+    while repoIter.%GetNext(.repoName, .repoVals) {
+        if ##class(%IPM.Repo.Definition).ServerDefinitionKeyExists(repoName) {
+            if (verbose) {
+                write !, "Repo: "_repoName_" already exists, skipping creating new one from lock file"
+            }
+            continue
+        }
+        elseif (verbose) {
+            write !, "Creating repo: "_repoName_" from lock file"
+        }
+        do ##class(%IPM.Repo.Definition).CollectServerTypes(.types)
+        set repoClass = types(repoVals.type)
+        set repoVals.name = repoName
+        do $classmethod(repoClass, "LockFileValuesToModifiers", repoVals, .modifiers)
+        $$$ThrowOnError($classmethod(repoClass,"Configure",0,.modifiers,.tData,repoClass))
+    }
+
+    // Install modules (even if they already exist)
+    do ..GetOrderedDependenciesList(dependencies, .orderedDependenciesList)
+    set depName = ""
+    for i=1:1:$listlength(orderedDependenciesList) {
+        set depName = $list(orderedDependenciesList, i)
+        set depVals = dependencies.%Get(depName)
+
+        set version = depVals.version
+        set repository = depVals.repository
+
+        // Call CleanInstall() on dependency modules but set flag "LockFileInstallStarted" so we don't try installing from the dependency module's lock file
+        set commandInfo = "ci"
+        set commandInfo("data", "Verbose") = verbose
+        set commandInfo("parameters","module") = repository_"/"_depName
+        set commandInfo("parameters", "version") = version
+        set commandInfo("data", "LockFileInstallStarted") = 1
+        do ##class(%IPM.Main).CleanInstall(.commandInfo)
+    }
+}
+
+/// The dependencies list in a lock file is listed alphabetically.
+/// This method compiles the dependencies and creates an ordered list such that no module is listed
+/// before one of its dependencies. Can then trace the list this outputs and install in order
+ClassMethod GetOrderedDependenciesList(
+	dependencies As %DynamicObject,
+	ByRef orderedDependenciesList As %List = "") [ Internal ]
+{
+    set depIter = dependencies.%GetIterator()
+    while depIter.%GetNext(.depName, .depVals) {
+        do ..AddToOrderedDependenciesList(depName, dependencies, .orderedDependenciesList)
+    }
+}
+
+/// For a dependency, recursively adds all dependencies to the ordered list, followed by this dependency
+ClassMethod AddToOrderedDependenciesList(
+	dependencyName As %String,
+	dependencies As %DynamicObject,
+	ByRef orderedDependenciesList As %List) [ Internal, Private ]
+{
+    set depVals = dependencies.%Get(dependencyName)
+    set transientDeps = depVals.%Get("dependencies", {})
+    set transientIter = transientDeps.%GetIterator()
+    while transientIter.%GetNext(.transDepName) {
+        // If dependency hasn't been installed yet, then recursively run this method on it
+        if '$listfind(orderedDependenciesList, transDepName) {
+            do ..AddToOrderedDependenciesList(transDepName, dependencies, .orderedDependenciesList)
+        }
+    }
+    if '$listfind(orderedDependenciesList, dependencyName) {
+        set orderedDependenciesList = orderedDependenciesList _ $listbuild(dependencyName)
+    }
+}
+
 }

src/cls/IPM/Lifecycle/Base.cls

@@ -1227,6 +1227,7 @@ Method %Export(
             "changelog.md",
             "license",
             "requirements.txt",
+            "module-lock.json",
         )
         set tRes = ##class(%File).FileSetFunc(..Module.Root)
         while tRes.%Next() {

src/cls/IPM/Main.cls

@@ -791,6 +791,25 @@ generate /my/path -export 00000,PacketName2,IgnorePacket2^00000,PacketName3,Igno
 	<example description="Show details of history item with ID 3 and information for each undergone lifecyle phase">history details 3 -phases</example>
 </command>
 
+<command name="ci">
+<summary>Installs a module from a lock file</summary>
+<description>
+    Installs a module from its lock file. Will first install all listed repositories followed by dependency modules and then the base module.
+</description>
+
+<!-- Examples -->
+<example description="Reads from the module-lock.json defined for mymodule and installs the repositories, module, and dependencies">
+    ci mymodule 3.0.0
+</example>
+
+<!-- Parameters -->
+<parameter name="module" required="true" description="Name of module on which to perform update actions" />
+<parameter name="version" description="Version (or version expression) of module to install; defaults to the latest available if unspecified." />
+
+<!-- Modifiers -->
+<modifier name="verbose" aliases="v" dataAlias="Verbose" dataValue="1" description="Produces verbose output from the command." />
+</command>
+
 </commands>
 }
 
@@ -1031,8 +1050,10 @@ ClassMethod ShellInternal(
                 do ..ModuleVersion(.tCommandInfo)
             } elseif (tCommandInfo = "information") {
                 do ..Information(.tCommandInfo)
-			} elseif (tCommandInfo = "history") {
-				do ..History(.tCommandInfo)
+            } elseif (tCommandInfo = "history") {
+                do ..History(.tCommandInfo)
+            } elseif (tCommandInfo = "ci") {
+                do ..CleanInstall(.tCommandInfo)
             }
         } catch pException {
             if (pException.Code = $$$ERCTRLC) {
@@ -2396,12 +2417,11 @@ ClassMethod Install(
                         $$$ThrowStatus($$$ERROR($$$GeneralError, "Deployed package '" _ tModuleName _ "' " _ tResult.VersionString _ " not supported on this platform " _ platformVersion _ "."))
                     }
                 }
-				$$$ThrowOnError(log.SetSource(tResult.ServerName))
-				$$$ThrowOnError(log.SetVersion(tResult.Version))
+                $$$ThrowOnError(log.SetSource(tResult.ServerName))
+                $$$ThrowOnError(log.SetVersion(tResult.Version))
                 $$$ThrowOnError(##class(%IPM.Utils.Module).LoadQualifiedReference(tResult, .tParams, , log))
             }
         } else {
-            set tPrefix = ""
             if (tModuleName '= "") {
                 if (tVersion '= "") {
                     $$$ThrowStatus($$$ERROR($$$GeneralError, tModuleName_" "_tVersion_" not found in any repository."))
@@ -2415,10 +2435,32 @@ ClassMethod Install(
             }
         }
     } catch ex {
-		$$$ThrowOnError(log.Finalize(ex.AsStatus(), devMode))
-		throw ex
-	}
-	$$$ThrowOnError(log.Finalize($$$OK, devMode))
+        $$$ThrowOnError(log.Finalize(ex.AsStatus(), devMode))
+        throw ex
+    }
+    $$$ThrowOnError(log.Finalize($$$OK, devMode))
+}
+
+ClassMethod CleanInstall(ByRef commandInfo) [ Internal ]
+{
+    set moduleName = $get(commandInfo("parameters","module"))
+    set version = $get(commandInfo("parameters","version"))
+    set verbose = $get(commandInfo("data","Verbose"))
+    set log = ##class(%IPM.General.HistoryTemp).CleanInstallInit(moduleName)
+
+    // TODO: Add "path"? (see Update() for more info of calling install v load)
+
+    if verbose {
+        write !, "Going to run a clean install on "_moduleName
+    }
+
+    // Indicating to commandInfo that this is a clean install command, not an install or load command. commandInfo will be passed to either Install() or Load() to continue performing the update.
+    set commandInfo("data","cmd") = "ci"
+    set commandInfo("data","CleanInstall") = 1
+    set log = ##class(%IPM.General.HistoryTemp).UpdateInit(moduleName)
+
+    // Forward execution to install
+    do ..Install(.commandInfo, log)
 }
 
 ClassMethod Reinstall(ByRef pCommandInfo) [ Internal ]

src/cls/IPM/Repo/Definition.cls

@@ -288,6 +288,16 @@ Method LockFileTypeGet()
     return ..#MONIKER
 }
 
+/// We use the "LockFileMapping" XData block to export a repo definition to a lock file.
+/// When importing from a lock file, doing an import and save using the same XData block won't work.
+/// Instead, we must create a set of modifiers and call %IPM.Repo.Definition::Configure()
+/// This method accepts the JSON repository values from a lock file and populates a modifers object to then call Configure() with
+ClassMethod LockFileValuesToModifiers(
+	lockFileValues As %DynamicObject,
+	Output modifiers) [ Abstract ]
+{
+}
+
 Storage Default
 {
 <Data name="RepoDefinitionDefaultData">

src/cls/IPM/Repo/Filesystem/Definition.cls

@@ -357,11 +357,21 @@ ClassMethod ScanDirectory(
     quit tSC
 }
 
+ClassMethod LockFileValuesToModifiers(
+	lockFileValues As %DynamicObject,
+	Output modifiers)
+{
+    set modifiers("filesystem") = ""
+    set modifiers("name") = lockFileValues.%Get("name")
+    set modifiers("path") = lockFileValues.%Get("root")
+    set modifiers("depth") = lockFileValues.%Get("depth")
+    set modifiers("read-only") = lockFileValues.%Get("readOnly")
+}
+
 XData LockFileMapping
 {
 <Mapping xmlns="http://www.intersystems.com/jsonmapping">
   <Property Name="LockFileType" FieldName="type" />
-  <Property Name="OverriddenSortOrder" FieldName="overriddenSortOrder" />
   <Property Name="ReadOnly" FieldName="readOnly" />
   <Property Name="Root" FieldName="root" />
   <Property Name="Depth" FieldName="depth" />

src/cls/IPM/Repo/Oras/Definition.cls

@@ -140,11 +140,34 @@ Method GetPublishingManager(ByRef status)
     return ##class(%IPM.Repo.Oras.PublishManager).%Get(.status)
 }
 
+ClassMethod LockFileValuesToModifiers(
+	lockFileValues As %DynamicObject,
+	Output modifiers)
+{
+    set modifiers("oras") = ""
+    set modifiers("name") = lockFileValues.%Get("name")
+    set modifiers("read-only") = lockFileValues.%Get("readOnly")
+    set modifiers("url") = lockFileValues.%Get("url")
+    set modifiers("namespace") = lockFileValues.%Get("orasNamespace")
+
+    // The following variables are set as system level variables for us to get
+    // Naming convention for those follow the name of the repository, first converting any '-' to '_',
+    // then removing everything except for alphabetic characters, numbers, and '_' to use as variable prefix
+    // lastly, adds a suffix based on the modifier
+    // Examples: <repo name>        - <prefix>
+    //           "registry"         - "registry"
+    //           "ORAS!Repo(5)?"    - "ORASRepo5"
+    //           "My-Repository-2"  - "My_Repository_2"
+    set prefix = $zstrip($replace(modifiers("name"), "-", "_"), "*E'N'A")
+    set modifiers("username") = $system.Util.GetEnviron(prefix_"_username")
+    set modifiers("password") = $system.Util.GetEnviron(prefix_"_password")
+    set modifiers("token") = $system.Util.GetEnviron(prefix_"_token")
+}
+
 XData LockFileMapping
 {
 <Mapping xmlns="http://www.intersystems.com/jsonmapping">
   <Property Name="LockFileType" FieldName="type" />
-  <Property Name="OverriddenSortOrder" FieldName="overriddenSortOrder" />
   <Property Name="ReadOnly" FieldName="readOnly" />
   <Property Name="URL" FieldName="url" />
   <Property Name="Namespace" FieldName="orasNamespace" />

src/cls/IPM/Repo/Remote/Definition.cls

@@ -132,6 +132,29 @@ Method GetPublishingManager(ByRef status)
     return ##class(%IPM.Repo.Remote.PublishManager).%Get(.status)
 }
 
+ClassMethod LockFileValuesToModifiers(
+	lockFileValues As %DynamicObject,
+	Output modifiers)
+{
+    set modifiers("remote") = ""
+    set modifiers("name") = lockFileValues.%Get("name")
+    set modifiers("url") = lockFileValues.%Get("url")
+    set modifiers("read-only") = lockFileValues.%Get("readOnly")
+
+    // The following variables are set as system level variables for us to get
+    // Naming convention for those follow the name of the repository, first converting any '-' to '_',
+    // then removing everything except for alphabetic characters, numbers, and '_' to use as variable prefix
+    // lastly, adds a suffix based on the modifier
+    // Examples: <repo name>        - <prefix>
+    //           "registry"         - "registry"
+    //           "ORAS!Repo(5)?"    - "ORASRepo5"
+    //           "My-Repository-2"  - "My_Repository_2"
+    set prefix = $zstrip($replace(modifiers("name"), "-", "_"), "*E'N'A")
+    set modifiers("username") = $system.Util.GetEnviron(prefix_"_username")
+    set modifiers("password") = $system.Util.GetEnviron(prefix_"_password")
+    set modifiers("token") = $system.Util.GetEnviron(prefix_"_token")
+}
+
 Method LockFileTypeGet()
 {
     return ..#MONIKERALIAS
@@ -141,7 +164,6 @@ XData LockFileMapping
 {
 <Mapping xmlns="http://www.intersystems.com/jsonmapping">
   <Property Name="LockFileType" FieldName="type" />
-  <Property Name="OverriddenSortOrder" FieldName="overriddenSortOrder" />
   <Property Name="ReadOnly" FieldName="readOnly" />
   <Property Name="URL" FieldName="url" />
 </Mapping>

src/cls/IPM/Utils/Module.cls

@@ -271,6 +271,9 @@ ClassMethod LoadModuleFromDirectory(
 {
     set tSC = $$$OK
     try {
+        if $get(pParams("CleanInstall"), 0) && '$get(pParams("LockFileInstallStarted"), 0) {
+            do ##class(%IPM.General.LockFile).InstallFromLockFile(pDirectory, .pParams)
+        }
         set tVerbose = $get(pParams("Verbose"))
         // LoadNewModule goes all the way through Reload->Validate->Compile->Activate, also compiling the new module.
         write:tVerbose !,"Loading from ",pDirectory,!
@@ -948,7 +951,6 @@ ClassMethod GetModuleNameFromXML(
 ///     <Parameter Name="NoLock">1</Parameter>
 /// </Defaults>
 /// ```
-/// 
 /// Returns results as multidimensional array
 ClassMethod GetModuleDefaultsFromXML(
 	pDirectory As %String,
@@ -1205,7 +1207,12 @@ ClassMethod LoadNewModule(
         if $get(params("CreateLockFile"), 0) && '$data(params("LockFileModule")){
             set params("LockFileModule") = tModule.Name
         }
-        do ..LoadDependencies(tModule,, .params)
+
+        // If installing from a lock file, don't need to load dependencies since dependencies will be installed in order anyways
+        if ('$get(params("CleanInstall"), 0)) {
+            do ..LoadDependencies(tModule, .params)
+        }
+
 
         set tSC = $system.OBJ.Load(pDirectory_"module.xml",$select(tVerbose:"d",1:"-d"),,.tLoadedList)
         $$$ThrowOnError(tSC)