diff --git a/.gitignore b/.gitignore index bf60b00a2..8031b0b0f 100644 --- a/.gitignore +++ b/.gitignore @@ -369,4 +369,7 @@ npm/*.tgz .DS_Store -.env \ No newline at end of file +.env + +*.lscache +dotnet/ \ No newline at end of file diff --git a/core/Enums/PackageType.cs b/core/Enums/PackageType.cs index 19eddbda7..457b7ee9c 100644 --- a/core/Enums/PackageType.cs +++ b/core/Enums/PackageType.cs @@ -89,5 +89,10 @@ public enum PackageType /// The Grafana /// Grafana = 16, + + /// + /// The Clickhouse Database + /// + ClickhouseDatabase = 17 } } diff --git a/core/Enums/StepType.cs b/core/Enums/StepType.cs index 51279e090..de272690d 100644 --- a/core/Enums/StepType.cs +++ b/core/Enums/StepType.cs @@ -123,6 +123,8 @@ public enum StepType SignEntityTypes = 35, - EnqueueXmla = 36 + EnqueueXmla = 36, + + GrantPermissions = 37 } } \ No newline at end of file diff --git a/core/Objects/CmfPackage/CmfPackageV1.cs b/core/Objects/CmfPackage/CmfPackageV1.cs index eb8558f07..df68d2a3b 100644 --- a/core/Objects/CmfPackage/CmfPackageV1.cs +++ b/core/Objects/CmfPackage/CmfPackageV1.cs @@ -7,6 +7,7 @@ using Cmf.CLI.Core.Interfaces; using Newtonsoft.Json; using Newtonsoft.Json.Converters; +using Newtonsoft.Json.Linq; namespace Cmf.CLI.Core.Objects; @@ -248,6 +249,57 @@ public class CmfPackageV1 : IEquatable /// [JsonProperty(Order = 23)] public string DependenciesDirectory { get; set; } + + /// + /// Gets the manifest version declared in the Deployment Framework manifest.xml. + /// Only used when converting a DF package (manifest.xml) to its package.json representation. + /// + [JsonIgnore] + public int ManifestVersion { get; internal set; } + + /// + /// Gets the minimum SQL Server compatibility level required by the package. + /// Only used when converting a DF package (manifest.xml) to its package.json representation. + /// + [JsonIgnore] + public int MinSqlCompatibility { get; internal set; } + + /// + /// Gets the target layer directory, which means the directory inside the container in which + /// the package contents should be installed when using Environment Manager. + /// Only used when converting a DF package (manifest.xml) to its package.json representation. + /// + [JsonIgnore] + public string TargetLayerDirectory { get; internal set; } + + /// + /// Gets the package build date declared in the Deployment Framework manifest.xml. + /// Only used when converting a DF package (manifest.xml) to its package.json representation. + /// + [JsonIgnore] + public DateTime? BuildDate { get; internal set; } + + /// + /// Gets the package upgrade strategy declared in the Deployment Framework manifest.xml. + /// Only used when converting a DF package (manifest.xml) to its package.json representation. + /// + [JsonIgnore] + public string UpgradeStrategy { get; internal set; } + + /// + /// Gets the extended metadata (e.g. application name/version and custom metadata entries) + /// declared in the Deployment Framework manifest.xml. + /// Only used when converting a DF package (manifest.xml) to its package.json representation. + /// + [JsonIgnore] + public Dictionary ExtendedMetadata { get; internal set; } + + /// + /// Gets the package demands declared in the Deployment Framework manifest.xml. + /// Only used when converting a DF package (manifest.xml) to its package.json representation. + /// + [JsonIgnore] + public List PackageDemands { get; internal set; } #endregion #region constructors diff --git a/core/Objects/Step.cs b/core/Objects/Step.cs index 67c4e4080..50a457d05 100644 --- a/core/Objects/Step.cs +++ b/core/Objects/Step.cs @@ -2,7 +2,10 @@ using Cmf.CLI.Utilities; using Newtonsoft.Json; using Newtonsoft.Json.Converters; +using Newtonsoft.Json.Linq; using System; +using System.Collections.Generic; +using System.Xml.Linq; using System.Xml.Serialization; namespace Cmf.CLI.Core.Objects @@ -13,6 +16,13 @@ namespace Cmf.CLI.Core.Objects /// public class Step : IEquatable { + #region Private Properties + + private List _elements; + private List _jElements; + + #endregion + #region Public Properties /// @@ -274,6 +284,13 @@ public class Step : IEquatable /// public string Value { get; set; } + /// + /// Gets the inner XML child elements of this step (e.g. from a Deployment Framework manifest.xml step, such as <Role> or <Object>). + /// Only used when converting a DF package (manifest.xml) to its package.json representation. + /// + [JsonIgnore] + public IEnumerable Elements => _elements; + #endregion #region Constructors @@ -334,6 +351,26 @@ public Step(StepType? type) #region Public Methods + /// + /// Adds an inner XML child element to this step (used when parsing a Deployment Framework manifest.xml step). + /// + /// The element. + public void AddElement(XElement element) + { + _elements ??= new List(); + _elements.Add(element); + } + + /// + /// Adds an inner JSON child element to this step (used when parsing a Deployment Framework package.json step). + /// + /// The element. + public void AddElement(JObject element) + { + _jElements ??= new List(); + _jElements.Add(element); + } + /// /// Indicates whether the current object is equal to another object of the same type. /// diff --git a/core/Services/CmfPackageController.cs b/core/Services/CmfPackageController.cs index 8a5cc093c..700f426d5 100644 --- a/core/Services/CmfPackageController.cs +++ b/core/Services/CmfPackageController.cs @@ -275,16 +275,11 @@ public async Task LoadDependencies(IEnumerable repoUris, StatusContext ctx, } - private static void LogUnknownAttributes(XElement element, IEnumerable knownAttributes = null) + private static List CheckForUnknownAttributes(XElement element, IEnumerable knownAttributes = null) { knownAttributes ??= []; var allAttributes = element.Attributes().Select(a => a.Name.LocalName); - - var unknownAttributes = allAttributes.Except(knownAttributes).ToList(); - if (unknownAttributes.Count > 0) - { - Log.Debug($"Step (type: {element.Attribute("type")?.Value}) has unknown attributes. Attributes: {string.Join(", ", unknownAttributes)}"); - } + return allAttributes.Except(knownAttributes).ToList(); } private static CmfPackageV1 FromXmlManifest(string manifest, bool setDefaultValues = false) @@ -378,19 +373,20 @@ public static CmfPackageV1 FromXml(XDocument xml) throw new CliException("Invalid manifest"); } - // if (rootNode.Element("systemName", true) != null) - // { - // package.AddMetadata(MetadataKey.ApplicationName, rootNode.Element("systemName", true).Value); - // - // var targetSystemVersionText = rootNode.Element("systemVersion", true)?.Value; - // if (!string.IsNullOrWhiteSpace(targetSystemVersionText)) - // { - // package.AddMetadata(MetadataKey.ApplicationVersion, targetSystemVersionText); - // } - // } + var strictStepParsing = Environment.GetEnvironmentVariable("cmf_cli_internal_strict_step_parsing") != null; + var parsingErrors = new List(); PackageType cliPackageType = PackageType.Generic; - Enum.TryParse(rootNode.Element("clipackagetype", true)?.Value, out cliPackageType); + if (!Enum.TryParse(rootNode.Element("clipackagetype", true)?.Value, out cliPackageType)) + { + if (!Enum.TryParse(rootNode.Element("packageType", true)?.Value, out cliPackageType)) + { + var logMsg = $"Unknown packageType: '{rootNode.Element("packageType", true)?.Value}'"; + parsingErrors.Add(logMsg); + if (strictStepParsing) Log.Error(logMsg); else Log.Debug(logMsg); + } + + } var steps = new List(); var stepsElements = rootNode.Element("steps", true)?.Elements("step", true); @@ -398,10 +394,26 @@ public static CmfPackageV1 FromXml(XDocument xml) { foreach (var element in stepsElements) { - LogUnknownAttributes(element, KnownStepAttributes); - + var unknownAttributes = CheckForUnknownAttributes(element, KnownStepAttributes); + var typeAttributeValue = element.Attribute("type")?.Value; + var typeParsed = Enum.TryParse(typeAttributeValue, out var parsedStepType); + + if (!typeParsed) + { + var logMsg = $"Step has an unknown type: {typeAttributeValue}"; + parsingErrors.Add(logMsg); + if (strictStepParsing) Log.Error(logMsg); else Log.Debug(logMsg); + } + + if (unknownAttributes.Count > 0) + { + var logMsg = $"Step (type: {typeAttributeValue}) has unknown attributes. Attributes: {string.Join(", ", unknownAttributes)}"; + parsingErrors.Add(logMsg); + if (strictStepParsing) Log.Error(logMsg); else Log.Debug(logMsg); + } + Step step = new Step { - Type = Enum.TryParse(element.Attribute("type")?.Value, out var parsedStepType) ? parsedStepType : StepType.Generic, + Type = typeParsed ? parsedStepType : StepType.Generic, Title = element.Attribute("title")?.Value, OnExecute = element.Attribute("onExecute")?.Value, ContentPath = element.Attribute("contentPath")?.Value, @@ -448,34 +460,19 @@ public static CmfPackageV1 FromXml(XDocument xml) // using var reader = element.CreateReader(); // // Deserialize the XML string into a Person object // Step s = (Step)serializer.Deserialize(reader); + foreach (var innerElement in element.Elements()) + { + step.AddElement(innerElement); + } + steps.Add(step); } } - // #region PackageDemands Parsing - // - // var packageDemands = rootNode.Element("packageDemands", true)?.Elements("packageDemand", true); - // if (packageDemands != null) - // { - // foreach (var item in packageDemands) - // { - // PackageDemand packageDemand = ParseDemand(item); - // - // package.AddDemand(packageDemand); - // } - // } - // - // #endregion - - // var variables = rootNode.Element("variables", true)?.Elements("variable", true); - // if (variables != null) - // { - // foreach (var element in variables) - // { - // var variable = ParseVariable(element); - // package.AddVariable(variable); - // } - // } + if (strictStepParsing && parsingErrors.Count > 0) + { + throw new CliException("CLI encountered unknown metadata when parsing package's manifest.xml! Check error messages for details."); + } DependencyCollection deps = new(); DependencyCollection testPackages = new(); @@ -542,40 +539,6 @@ public static CmfPackageV1 FromXml(XDocument xml) } } - // var uiElements = rootNode.Element("ui", true)?.Elements("wizardStep", true); - // if (uiElements != null) - // { - // foreach (var uiElement in uiElements) - // { - // var step = (PackageWizardStep)ParseGroup(uiElement, new PackageWizardStep()); - // step.Id = uiElement.Attribute("id")?.Value; - // package.UserInterface.AddStep(step); - // } - // } - - // var metadataElements = rootNode.Element("metadata", true)?.Elements(); - // if (metadataElements != null) - // { - // foreach (var element in metadataElements) - // { - // package.AddMetadata(element.Name.LocalName, element.Value); - // } - // } - - // string manifestVersionString = rootNode.Element("manifestVersion", true)?.Value; - // int manifestVersion = -1; - // Int32.TryParse(manifestVersionString, out manifestVersion); - // package.ManifestVersion = manifestVersion; - // - // string minSqlCompatibilityString = rootNode.Element("minSqlCompatibility", true)?.Value; - // int minSqlCompatibility = -1; - // Int32.TryParse(minSqlCompatibilityString, out minSqlCompatibility); - // package.MinSqlCompatibility = minSqlCompatibility; - // package.TargetLayerDirectory = rootNode.Element("targetLayerDirectory", true)?.Value; - // package.IsToForceInstall = rootNode.Element("IsToForceInstall", true)?.Value != null ? bool.Parse(rootNode.Element("IsToForceInstall", true).Value) : false; - // package.BuildDate = rootNode.Element("buildDate", true)?.Value != null ? DateTime.ParseExact(rootNode.Element("buildDate", true).Value, "dd/MM/yyyy", System.Globalization.CultureInfo.InvariantCulture) : (DateTime?)null; - // package.IsRootPackage = (rootNode.Element("keywords", true)?.Value.Split(',').Any(k => k.Contains("cmf-root-package")) ?? false); - // package.ForceRerunAfterDatabaseRestore = bool.Parse(rootNode.Element("forceRerunAfterDatabaseRestore", true)?.Value ?? "false"); var cmfPackage = new CmfPackageV1( rootNode.Element("name", true)?.Value, rootNode.Element("packageId", true)?.Value, @@ -597,7 +560,74 @@ public static CmfPackageV1 FromXml(XDocument xml) waitForIntegrationEntries: false, testPackages ); - + + // Additional Deployment Framework metadata. These fields are not part of the source + // cmfpackage.json, but they are required to achieve package.json parity with Environment + // Manager when converting a DF package (manifest.xml) to its npm (package.json) representation. + cmfPackage.ManifestVersion = int.TryParse(rootNode.Element("manifestVersion", true)?.Value, out var manifestVersion) ? manifestVersion : 0; + cmfPackage.MinSqlCompatibility = int.TryParse(rootNode.Element("minSqlCompatibility", true)?.Value, out var minSqlCompatibility) ? minSqlCompatibility : 0; + cmfPackage.TargetLayerDirectory = rootNode.Element("targetLayerDirectory", true)?.Value; + cmfPackage.BuildDate = rootNode.Element("buildDate", true)?.Value != null + ? DateTime.ParseExact(rootNode.Element("buildDate", true).Value, "dd/MM/yyyy", System.Globalization.CultureInfo.InvariantCulture) + : null; + cmfPackage.UpgradeStrategy = rootNode.Element("upgradeStrategy", true)?.Value; + + // Extended metadata: target application info plus any custom entries. + var extendedMetadata = new Dictionary(); + var systemNameElement = rootNode.Element("systemName", true); + if (systemNameElement != null) + { + extendedMetadata["ApplicationName"] = systemNameElement.Value; + var systemVersion = rootNode.Element("systemVersion", true)?.Value; + if (!string.IsNullOrWhiteSpace(systemVersion)) + { + extendedMetadata["ApplicationVersion"] = systemVersion; + } + } + var metadataElements = rootNode.Element("metadata", true)?.Elements(); + if (metadataElements != null) + { + foreach (var element in metadataElements) + { + extendedMetadata[element.Name.LocalName] = element.Value; + } + } + cmfPackage.ExtendedMetadata = extendedMetadata; + + // Package demands: each element is parsed to a JObject with all its + // XML attributes as properties and any inner child elements grouped as named arrays + // (parity with Environment Manager's PackageManifestReader). + var parsedDemands = new List(); + var packageDemandsElements = rootNode.Element("packageDemands", true)?.Elements("packageDemand", true); + if (packageDemandsElements != null) + { + foreach (var element in packageDemandsElements) + { + var demandObject = new JObject(); + foreach (var attr in element.Attributes()) + { + demandObject[attr.Name.LocalName] = attr.Value; + } + var childGroups = element.Elements().GroupBy(e => e.Name.LocalName); + foreach (var group in childGroups) + { + var childArray = new JArray(); + foreach (var child in group) + { + var childObj = new JObject(); + foreach (var attr in child.Attributes()) + { + childObj[attr.Name.LocalName] = attr.Value; + } + childArray.Add(childObj); + } + demandObject[group.Key] = childArray; + } + parsedDemands.Add(demandObject); + } + } + cmfPackage.PackageDemands = parsedDemands; + return cmfPackage; } @@ -757,6 +787,15 @@ public static CmfPackageV1 FromJson(JObject json) relativePath: null, filePath: elem.Property("filePath")?.Value.ToString() ); + + foreach (var innerElement in elem.Children()) + { + if (innerElement.Type == JTokenType.Object) + { + step.AddElement((JObject)innerElement); + } + } + steps.Add(step); } } @@ -936,20 +975,13 @@ public string ToJson(bool lowercase = false, bool addManifestVersion = false) foreach (var step in package.Steps ?? Array.Empty().ToList()) { var stepObject = JObject.FromObject(step, jsonSerializer); - // foreach (var attributeName in step.AttributeNames) - // { - // if (stepObject[attributeName] == null) - // { - // stepObject.Add(attributeName, step.GetAttribute(attributeName)); - // } - // } - // - // var elementTypes = step.Elements.ToLookup(x => x.Name.LocalName); - // - // foreach (var element in elementTypes) - // { - // stepObject.Add(element.Key, JArray.FromObject(element.ToArray(), jsonSerializer)); - // } + + var elementTypes = (step.Elements ?? Enumerable.Empty()).ToLookup(x => x.Name.LocalName); + + foreach (var element in elementTypes) + { + stepObject.Add(element.Key, JArray.FromObject(element.ToArray(), jsonSerializer)); + } stepsArray.Add(stepObject); } @@ -999,33 +1031,6 @@ public string ToJson(bool lowercase = false, bool addManifestVersion = false) JObject j = new JObject(new JProperty("steps", stepsArray)); #endregion - #region assemble wizardSteps - JArray uiArray = new JArray(); - JArray variablesArray = new JArray(); - - var jsonSerializerSettingsUI = new JsonSerializerSettings() - { - NullValueHandling = NullValueHandling.Ignore, - ContractResolver = new CamelCasePropertyNamesContractResolver(), - PreserveReferencesHandling = PreserveReferencesHandling.None, - }; - - // var jsonSerializerUI = JsonSerializer.Create(jsonSerializerSettingsUI); - // - // foreach (var wizardStep in package.UserInterface.Steps) - // { - // var wizardStepObject = JObject.FromObject(wizardStep, jsonSerializerUI); - // uiArray.Add(wizardStepObject); - // } - // - // foreach (var variable in package.Variables) - // { - // var variablesObject = JObject.FromObject(variable, jsonSerializerUI); - // variablesArray.Add(variablesObject); - // } - - #endregion - #region assemble dependencies var dependecies = new JObject(); var mandatoryDependecies = new JObject(); @@ -1099,11 +1104,13 @@ string getUniquePackageAlias(string packageName) #region assemble metadata var metadata = new JObject(); - // foreach (var meta in package.extendedMetadata) - // { - // var property = new JProperty(meta.Key, meta.Value); - // metadata.Add(property); - // } + if (package.ExtendedMetadata != null) + { + foreach (var meta in package.ExtendedMetadata) + { + metadata.Add(new JProperty(meta.Key, meta.Value)); + } + } #endregion #region assemble keywords @@ -1118,10 +1125,15 @@ string getUniquePackageAlias(string packageName) { keywordsList.Add(JSONPackageKeywordIsInstallable); } - // if (package.IsRootPackage) - // { - // keywordsList.Add(JSONPackageKeywordIsRootPackage); - // } + + // Mark root packages so they can be discovered on a package registry + var isRootPackage = package.Keywords? + .Split(',') + .Any(k => k.Contains(CoreConstants.RootPackageDefaultKeyword)) ?? false; + if (isRootPackage) + { + keywordsList.Add(JSONPackageKeywordIsRootPackage); + } var keywords = JArray.FromObject(keywordsList); @@ -1131,67 +1143,23 @@ string getUniquePackageAlias(string packageName) JArray demandsArray = new JArray(); - // foreach (var demand in package.Demands) - // { - // var demandObject = JObject.FromObject(demand, jsonSerializer); - // foreach (var attributeName in demand.AttributeNames) - // { - // demandObject.Add(attributeName, demand.GetAttribute(attributeName)); - // } - // - // var elementTypes = demand.Elements.ToLookup(x => x.Name.LocalName); - // - // foreach (var element in elementTypes) - // { - // demandObject.Add(element.Key, JArray.FromObject(element.ToArray(), jsonSerializer)); - // } - // - // demandsArray.Add(demandObject); - // } - - - foreach (JObject desc in demandsArray) + if (package.PackageDemands != null) { - var children = desc.Value(); - - if (children.Properties() != null) + foreach (var demand in package.PackageDemands) { - var values = children.Value(); - foreach (var value in values) - { - if (value.Value.Type == JTokenType.Array) - { - var entityTypes = value.Value; - var entities = entityTypes.Values(); - foreach (var entity in entities) - { - var entityValueAux = entity.Value(); - var entityValues = entityValueAux.Value(); - foreach (var entityValue in entityValues) - { - var final = entityValue.Value.Value(); - foreach (var finalElement in final) - { - if (finalElement.Key.StartsWith("@")) - { - var newValue = finalElement.Key.Replace("@", ""); - JProperty newProperty = new JProperty(newValue, finalElement.Value); - jObject.Add(newProperty); - } - } - final.RemoveAll(); - final.Add(jObject.Properties()); - jObject.RemoveAll(); - } - } - } - } + demandsArray.Add(demand); } - } #endregion + // manifestVersion is always present in the deployment object (parity with Environment Manager). + // Prefer the value declared in the manifest.xml; when it is absent, fall back to the CLI default + // if the caller requested it (preserving previous behavior), otherwise keep the parsed value. + var manifestVersion = package.ManifestVersion > 0 + ? package.ManifestVersion + : (addManifestVersion ? CoreConstants.ManifestVersion : package.ManifestVersion); + JObject jsonObject = new JObject( new JProperty("name", lowercase ? package.PackageId.ToLowerInvariant() : package.PackageId), new JProperty("description", package.Description), @@ -1201,15 +1169,16 @@ string getUniquePackageAlias(string packageName) new JProperty("keywords", keywords), new JProperty("isToForceInstall", package.IsToForceInstall), new JProperty("isUniqueInstall", package.IsUniqueInstall), - new JProperty("forceRerunAfterDatabaseRestore", package.ForceRerunAfterDatabaseRestore), + new JProperty("forceRerunAfterDatabaseRestore", package.ForceRerunAfterDatabaseRestore ?? false), + new JProperty("upgradeStrategy", package.UpgradeStrategy ?? string.Empty), new JProperty("deployment", new JObject( - // new JProperty("manifestVersion", package.ManifestVersion), + new JProperty("manifestVersion", manifestVersion), new JProperty("isInstallable", package.IsInstallable), - new JProperty("packageType", package.PackageType), + new JProperty("packageType", Enum.GetName(package.PackageType)), new JProperty("targetDirectory", !String.IsNullOrEmpty(package.TargetDirectory) ? package.TargetDirectory : ""), - // new JProperty("targetLayerDirectory", !String.IsNullOrEmpty(package.TargetLayerDirectory) ? package.TargetLayerDirectory : ""), + new JProperty("targetLayerDirectory", !String.IsNullOrEmpty(package.TargetLayerDirectory) ? package.TargetLayerDirectory : ""), new JProperty("targetLayer", !String.IsNullOrEmpty(package.TargetLayer) ? package.TargetLayer : ""), - // new JProperty("buildDate", package.BuildDate?.ToString()), + new JProperty("buildDate", package.BuildDate?.ToString()), new JProperty("steps", stepsArray), new JProperty("packageDemands", demandsArray))), new JProperty("dependencies", dependecies), @@ -1217,32 +1186,11 @@ string getUniquePackageAlias(string packageName) new JProperty("conditionalDependencies", conditionalDependencies), new JProperty("_originalPackageId", package.PackageId)); - if (addManifestVersion) + if (package.MinSqlCompatibility > 0) { - jsonObject.SelectToken("deployment")?.Value()?.AddFirst(new JProperty("manifestVersion", CoreConstants.ManifestVersion)); + jsonObject["deployment"]["minSqlCompatibility"] = package.MinSqlCompatibility; } - // if (package.MinSqlCompatibility > 0) - // { - // jsonObject["deployment"]["minSqlCompatibility"] = package.MinSqlCompatibility; - // } - - if (uiArray.Count > 0) - { - var uiProperty = new JProperty("ui", uiArray); - var jsonFinal = (JObject)jsonObject.Property("deployment").Value; - jsonFinal.Add(uiProperty); - jsonObject.Property("deployment").Value = jsonFinal; - - - } - if (variablesArray.Count > 0) - { - var varProperty = new JProperty("variables", variablesArray); - var jsonFinal = (JObject)jsonObject.Property("deployment").Value; - jsonFinal.Add(varProperty); - jsonObject.Property("deployment").Value = jsonFinal; - } if (metadata.HasValues) { var metadataProperty = new JProperty("metadata", metadata); @@ -1255,243 +1203,6 @@ string getUniquePackageAlias(string packageName) } #region utils - // private static PackageWizardStepGroup ParseGroup(JObject uiElement, PackageWizardStepGroup result = null) - // { - // result = result ?? new PackageWizardStepGroup(); - // - // result.Id = uiElement.Property("id")?.Value.ToString(); - // result.Order = int.Parse(uiElement.Property("order")?.Value.ToString() ?? "0"); - // result.Title = uiElement.Property("title")?.Value.ToString(); - // result.Type = uiElement.Property("type")?.Value.ToString(); - // result.RequiresValidation = bool.Parse(uiElement.Property("requiresValidation")?.Value.ToString() ?? "false"); - // result.IsAdvanced = bool.Parse(uiElement.Property("isAdvanced")?.Value.ToString() ?? "false"); - // result.Condition = uiElement.Property("condition")?.Value.ToString(); - // - // var groupElements = uiElement.Property("groups")?.Value; - // if (groupElements?.Type == JTokenType.Array) - // { - // var groupArray = (JArray)groupElements; - // if (groupArray != null) - // { - // foreach (JObject groupElement in groupArray) - // { - // var group = ParseGroup(groupElement); - // result.Groups.Add(group); - // } - // } - // } - // - // var variableElements = uiElement.Property("variables")?.Value; - // if (variableElements?.Type == JTokenType.Array) - // { - // var variablesArray = (JArray)variableElements; - // if (variablesArray != null) - // { - // foreach (JObject variableElement in variablesArray) - // { - // var variable = ParseVariable(variableElement); - // result.Variables.Add(variable); - // } - // } - // } - // - // return result; - // } - // - // private static VariableDefinition ParseVariable(XElement element) - // { - // var result = new VariableDefinition - // { - // Name = element.Attribute("name")?.Value, - // ValueType = element.Attribute("valueType")?.Value, - // IsRequired = bool.Parse(element.Attribute("isRequired")?.Value ?? "false"), - // GroupName = element.Attribute("groupName")?.Value, - // Label = element.Attribute("label")?.Value, - // ReadOnly = bool.Parse(element.Attribute("readOnly")?.Value ?? "false"), - // ValidationConfiguration = element.Attribute("validationConfiguration")?.Value, - // Placeholder = element.Attribute("placeholder")?.Value, - // Default = element.Attribute("default")?.Value, - // IsToValidate = bool.Parse(element.Attribute("isToValidate")?.Value ?? "true"), - // }; - // - // return result; - // } - // - // private static VariableDefinition ParseVariable(JObject element) - // { - // var result = new VariableDefinition - // { - // Name = element.Property("name")?.Value.ToString(), - // ValueType = element.Property("valueType")?.Value.ToString(), - // IsRequired = bool.Parse(element.Property("isRequired")?.Value.ToString() ?? "false"), - // GroupName = element.Property("groupName")?.Value.ToString(), - // Label = element.Property("label")?.Value.ToString(), - // ReadOnly = bool.Parse(element.Property("readOnly")?.Value.ToString() ?? "false"), - // ValidationConfiguration = element.Property("validationConfiguration")?.Value.ToString(), - // Placeholder = element.Property("placeholder")?.Value.ToString(), - // Default = element.Property("default")?.Value.ToString(), - // IsToValidate = bool.Parse(element.Property("isToValidate")?.Value.ToString() ?? "true"), - // }; - // - // return result; - // } - // - // - // private static PackageStep ParseStep(JObject element) - // { - // var coreAttributes = new string[] - // { - // "type", - // "id", - // "title", - // "onInitialize", - // "onAquire", - // "onValidate", - // "onPrepare", - // "onExecute", - // "onComplete", - // "onCleanup", - // "packageIds", - // "contentPath" - // }; - // - // PackageStep step; - // - // if (element.Property("reevaluatePlan") != null || element.Property("packageId") != null) - // { - // var packageDeploymentStep = new PackageDeploymentStep(); - // - // packageDeploymentStep.ReevaluatePlan = bool.Parse(element.Property("reevaluatePlan")?.Value.ToString() ?? "false"); - // packageDeploymentStep.PackageId = element.Property("packageId")?.Value.ToString(); - // step = packageDeploymentStep; - // } - // else - // { - // step = new PackageStep(); - // } - // - // step.Type = element.Property("type")?.Value.ToString(); - // step.Id = element.Property("id")?.Value.ToString(); - // step.Title = element.Property("title")?.Value.ToString(); - // step.ContentPath = element.Property("contentPath")?.Value.ToString(); - // step.OnInitialize = element.Property("onInitialize")?.Value.ToString(); - // step.OnAquire = element.Property("onAquire")?.Value.ToString(); - // step.OnValidate = element.Property("onValidate")?.Value.ToString(); - // step.OnPrepare = element.Property("onPrepare")?.Value.ToString(); - // step.OnExecute = element.Property("onExecute")?.Value.ToString(); - // step.OnComplete = element.Property("onComplete")?.Value.ToString(); - // step.OnCleanup = element.Property("onCleanup")?.Value.ToString(); - // - // var extraAttributes = element.Properties().Where(a => !coreAttributes.Contains(a.Name)).ToList(); - // foreach (var a in extraAttributes) - // { - // step.AddAttribute(a.Name, a.Value.ToString()); - // } - // - // foreach (var innerElement in element.Children()) - // { - // if (innerElement.Type == JTokenType.Object) - // { - // var elem = (JObject)innerElement; - // step.AddElement(elem); - // } - // } - // - // return step; - // } - - - // /// - // /// Parse PackageDemand elements from a XElement. - // /// - // /// Element to be parsed. - // /// - // private static PackageDemand ParseDemand(XElement element) - // { - // var coreAttributes = new string[] - // { - // "type", - // "id", - // "title", - // "value", - // "onInitialize", - // "onAquire", - // "onValidate", - // "onPrepare", - // "onExecute", - // "onComplete", - // "onCleanup", - // "packageIds" - // }; - // - // PackageDemand demand = new PackageDemand(); - // - // demand.Type = (PackageDemandType)Enum.Parse(typeof(PackageDemandType), element.Attribute("type")?.Value); - // demand.Id = element.Attribute("id")?.Value; - // demand.Title = element.Attribute("title")?.Value; - // demand.Value = element.Attribute("value")?.Value; - // - // var extraAttributes = element.Attributes().Where(a => !coreAttributes.Contains(a.Name.LocalName)).ToList(); - // foreach (var a in extraAttributes) - // { - // demand.AddAttribute(a.Name.LocalName, a.Value); - // } - // - // foreach (var innerElement in element.Elements()) - // { - // demand.AddElement(innerElement); - // } - // - // return demand; - // } - // - // /// - // /// Parse PackageDemand elements from a JObject. - // /// - // /// Element to be parsed. - // /// - // private static PackageDemand ParseDemand(JObject element) - // { - // var coreAttributes = new string[] - // { - // "type", - // "id", - // "title", - // "value", - // "onInitialize", - // "onAquire", - // "onValidate", - // "onPrepare", - // "onExecute", - // "onComplete", - // "onCleanup", - // "packageIds" - // }; - // - // PackageDemand demand = new PackageDemand(); - // - // demand.Type = (PackageDemandType)Enum.Parse(typeof(PackageDemandType), element.Property("type")?.Value.ToString()); - // demand.Id = element.Property("id")?.Value.ToString(); - // demand.Title = element.Property("title")?.Value.ToString(); - // demand.Value = element.Property("value")?.Value.ToString(); - // - // var extraAttributes = element.Properties().Where(a => !coreAttributes.Contains(a.Name)).ToList(); - // foreach (var a in extraAttributes) - // { - // demand.AddAttribute(a.Name, a.Value.ToString()); - // } - // - // foreach (var innerElement in element.Children()) - // { - // if (innerElement.Type == JTokenType.Object) - // { - // var elem = (JObject)innerElement; - // demand.AddElement(elem); - // } - // } - // - // return demand; - // } /// /// Convert json array of dependecies to list diff --git a/tests/Specs/CmfPackageController_FromXml.cs b/tests/Specs/CmfPackageController_FromXml.cs index e71b1fc31..3488b7e9a 100644 --- a/tests/Specs/CmfPackageController_FromXml.cs +++ b/tests/Specs/CmfPackageController_FromXml.cs @@ -1,8 +1,11 @@ +using System; using System.Linq; using System.Xml.Linq; using Cmf.CLI.Core.Enums; using Cmf.CLI.Core.Services; +using Cmf.CLI.Utilities; using FluentAssertions; +using Newtonsoft.Json.Linq; using Xunit; namespace tests.Specs; @@ -414,4 +417,386 @@ public void FromXml_ShouldParseConfigPathAndValue_ForUpdateConfigurationStep() pkg.Steps.Single().ConfigPath.Should().Be("./appsettings.json"); pkg.Steps.Single().Value.Should().Be("newValue"); } + + [Fact] + public void FromXml_ShouldDefaultForceRerunAfterDatabaseRestoreToFalse_WhenMissing() + { + var xml = XDocument.Parse( + """ + + Cmf.Custom.Data + 1.0.0 + + """); + + var pkg = CmfPackageController.FromXml(xml); + + pkg.ForceRerunAfterDatabaseRestore.Should().BeFalse(); + } + + [Fact] + public void FromXml_ShouldParseUpgradeStrategy_FromManifestElement() + { + var xml = XDocument.Parse( + """ + + Cmf.Custom.Data + 1.0.0 + CumulativeByMinor + + """); + + var pkg = CmfPackageController.FromXml(xml); + + pkg.UpgradeStrategy.Should().Be("CumulativeByMinor"); + } + + [Fact] + public void FromXml_ShouldParseManifestVersion_FromManifestElement() + { + var xml = XDocument.Parse( + """ + + Cmf.Custom.Data + 1.0.0 + 1 + + """); + + var pkg = CmfPackageController.FromXml(xml); + + pkg.ManifestVersion.Should().Be(1); + } + + [Fact] + public void FromXml_ShouldDefaultManifestVersionToZero_WhenMissing() + { + var xml = XDocument.Parse( + """ + + Cmf.Custom.Data + 1.0.0 + + """); + + var pkg = CmfPackageController.FromXml(xml); + + pkg.ManifestVersion.Should().Be(0); + } + + [Fact] + public void FromXml_ShouldParseMinSqlCompatibility_FromManifestElement() + { + var xml = XDocument.Parse( + """ + + Cmf.Custom.Data + 1.0.0 + 150 + + """); + + var pkg = CmfPackageController.FromXml(xml); + + pkg.MinSqlCompatibility.Should().Be(150); + } + + [Fact] + public void FromXml_ShouldParseTargetLayerDirectory_FromManifestElement() + { + var xml = XDocument.Parse( + """ + + Cmf.Custom.Data + 1.0.0 + some/layer/dir + + """); + + var pkg = CmfPackageController.FromXml(xml); + + pkg.TargetLayerDirectory.Should().Be("some/layer/dir"); + } + + [Fact] + public void FromXml_ShouldParseBuildDate_FromManifestElement() + { + var xml = XDocument.Parse( + """ + + Cmf.Custom.Data + 1.0.0 + 25/12/2024 + + """); + + var pkg = CmfPackageController.FromXml(xml); + + pkg.BuildDate.Should().Be(new DateTime(2024, 12, 25)); + } + + [Fact] + public void FromXml_ShouldParseExtendedMetadata_FromSystemNameVersionAndCustomMetadata() + { + var xml = XDocument.Parse( + """ + + Cmf.Custom.Data + 1.0.0 + MyApp + 11.1.0 + + customValue + + + """); + + var pkg = CmfPackageController.FromXml(xml); + + pkg.ExtendedMetadata.Should().ContainKey("ApplicationName"); + pkg.ExtendedMetadata["ApplicationName"].Should().Be("MyApp"); + pkg.ExtendedMetadata.Should().ContainKey("ApplicationVersion"); + pkg.ExtendedMetadata["ApplicationVersion"].Should().Be("11.1.0"); + pkg.ExtendedMetadata.Should().ContainKey("customKey"); + pkg.ExtendedMetadata["customKey"].Should().Be("customValue"); + } + + [Fact] + public void FromXml_ShouldParsePackageDemands_WithAttributesOnly() + { + var xml = XDocument.Parse( + """ + + Cmf.Custom.Data + 1.0.0 + + + + + """); + + var pkg = CmfPackageController.FromXml(xml); + + pkg.PackageDemands.Should().ContainSingle(); + pkg.PackageDemands[0]["type"]!.Value().Should().Be("SQLServer"); + pkg.PackageDemands[0]["version"]!.Value().Should().Be("14.0.1000.169"); + pkg.PackageDemands[0]["description"]!.Value().Should().Be("Microsoft SQL Server 2017"); + } + + [Fact] + public void FromXml_ShouldParsePackageDemands_WithInnerChildElements() + { + var xml = XDocument.Parse( + """ + + Cmf.Custom.Data + 1.0.0 + + + + + + """); + + var pkg = CmfPackageController.FromXml(xml); + + pkg.PackageDemands.Should().ContainSingle(); + pkg.PackageDemands[0]["type"]!.Value().Should().Be("Generic"); + var options = pkg.PackageDemands[0]["Option"] as JArray; + options.Should().HaveCount(2); + options![0]["name"]!.Value().Should().Be("opt1"); + options![1]["name"]!.Value().Should().Be("opt2"); + } + + [Fact] + public void FromXml_ShouldReturnEmptyPackageDemands_WhenNoDemandsDeclared() + { + var xml = XDocument.Parse( + """ + + Cmf.Custom.Data + 1.0.0 + + """); + + var pkg = CmfPackageController.FromXml(xml); + + pkg.PackageDemands.Should().BeEmpty(); + } + + [Fact] + public void FromXml_ShouldParseStepInnerElements_WithSingleRoleElement() + { + var xml = XDocument.Parse( + """ + + Cmf.Custom.Data + 1.0.0 + + + + + + + """); + + var pkg = CmfPackageController.FromXml(xml); + + var step = pkg.Steps.Should().ContainSingle().Subject; + step.Elements.Should().ContainSingle(); + step.Elements.Single().Name.LocalName.Should().Be("Role"); + step.Elements.Single().Attribute("name")?.Value.Should().Be("db_datareader"); + } + + [Fact] + public void FromXml_ShouldParseStepInnerElements_WithMultipleObjectElements() + { + var xml = XDocument.Parse( + """ + + Cmf.Custom.Data + 1.0.0 + + + + + + + + + """); + + var pkg = CmfPackageController.FromXml(xml); + + var step = pkg.Steps.Should().ContainSingle().Subject; + step.Type.Should().Be(StepType.GrantPermissions); + step.Elements.Should().HaveCount(3); + step.Elements.Select(e => e.Attribute("type")?.Value).Should().BeEquivalentTo(new[] { "udtt", "function", "procedure" }); + } + + [Fact] + public void FromXml_ShouldLeaveStepElementsNull_WhenStepHasNoInnerElements() + { + var xml = XDocument.Parse( + """ + + Cmf.Custom.Data + 1.0.0 + + + + + """); + + var pkg = CmfPackageController.FromXml(xml); + + pkg.Steps.Should().ContainSingle(); + pkg.Steps.Single().Elements.Should().BeNull(); + } + + [Fact] + public void ToJson_ShouldGroupStepInnerElements_ByElementName() + { + var xml = XDocument.Parse( + """ + + Cmf.Custom.Data + 1.0.0 + + + + + + + + """); + + var pkg = CmfPackageController.FromXml(xml); + var json = new CmfPackageController(pkg, null).ToJson(); + var jsonObject = JObject.Parse(json); + + var stepObject = (JObject)jsonObject["deployment"]["steps"][0]; + var objects = (JArray)stepObject["Object"]; + objects.Should().HaveCount(2); + ((JObject)objects[0]["Object"])["type"]!.Value().Should().Be("udtt"); + ((JObject)objects[1]["Object"])["type"]!.Value().Should().Be("function"); + } + + [Fact] + public void FromXml_ShouldThrowException_WhenStrictStepParsingEnabledAndStepHasUnknownAttribute() + { + Environment.SetEnvironmentVariable("cmf_cli_internal_strict_step_parsing", "1"); + try + { + var xml = XDocument.Parse( + """ + + Cmf.Custom.Data + 1.0.0 + + + + + """); + + Action act = () => CmfPackageController.FromXml(xml); + + act.Should().Throw().WithMessage("*CLI encountered unknown metadata*"); + } + finally + { + Environment.SetEnvironmentVariable("cmf_cli_internal_strict_step_parsing", null); + } + } + + [Fact] + public void FromXml_ShouldThrowException_WhenStrictStepParsingEnabledAndStepTypeIsInvalid() + { + Environment.SetEnvironmentVariable("cmf_cli_internal_strict_step_parsing", "1"); + try + { + var xml = XDocument.Parse( + """ + + Cmf.Custom.Data + 1.0.0 + + + + + """); + + Action act = () => CmfPackageController.FromXml(xml); + + act.Should().Throw().WithMessage("*CLI encountered unknown metadata*"); + } + finally + { + Environment.SetEnvironmentVariable("cmf_cli_internal_strict_step_parsing", null); + } + } + + [Fact] + public void FromXml_ShouldDefaultToGenericAndNotThrow_WhenStrictStepParsingDisabled() + { + Environment.SetEnvironmentVariable("cmf_cli_internal_strict_step_parsing", null); + + var xml = XDocument.Parse( + """ + + Cmf.Custom.Data + 1.0.0 + + + + + """); + + var pkg = CmfPackageController.FromXml(xml); + + pkg.Steps.Should().ContainSingle(); + pkg.Steps.Single().Type.Should().Be(StepType.Generic); + } } diff --git a/tests/Specs/Repositories.cs b/tests/Specs/Repositories.cs index 2596aa7fa..af1b03660 100644 --- a/tests/Specs/Repositories.cs +++ b/tests/Specs/Repositories.cs @@ -809,6 +809,94 @@ public void ConvertZipToTgz_WithPackageJson() Assert.Equal(packageRoot.Key, packageJson._originalPackageId.ToString()); } + [Fact] + public void ConvertZipToTgz_ShouldIncludeDeploymentMetadata_FromManifest() + { + KeyValuePair package = new("Cmf.Database.Runtime", "12.0.0"); + + string manifestContent = + @$" + {package.Key} + Critical Manufacturing Database Server Runtime + ProductDatabase + This package deploys Critical Manufacturing MES Database Server Runtime + 1 + 150 + {package.Value} + true + true + db/runtime + 25/12/2024 + cmf-root-package + true + CumulativeByMinor + MyApp + 11.1.0 + + customValue + + + + + + + + "; + + var fileSystem = new MockFileSystem(new Dictionary + { + { $"repo/{package.Key}.{package.Value}.zip", new MockFileData(new DFPackageBuilder().CreateEntry("manifest.xml", manifestContent).ToByteArray()) } + }); + ExecutionContext.Initialize(fileSystem); + + IFileInfo tgzPackageFile = fileSystem.FileInfo.New($"repo/{package.Key}.{package.Value}.tgz"); + + // test conversion zip->tgz + CmfPackageController.ConvertZipToTarGz(fileSystem.FileInfo.New($"repo/{package.Key}.{package.Value}.zip"), tgzPackageFile); + + // get tgz info for assertion + using GZipStream gzipStream = new GZipStream(tgzPackageFile.OpenRead(), CompressionMode.Decompress); + using TarReader tarReader = new(gzipStream); + dynamic packageJson = null; + while (tarReader.GetNextEntry() is { } entry) + { + if (entry.Name == "package/package.json") + { + using MemoryStream ms = new(); + entry.DataStream.CopyTo(ms); + packageJson = JsonConvert.DeserializeObject(Encoding.UTF8.GetString(ms.ToArray())); + } + } + + Assert.NotNull(packageJson); + + // top-level deployment metadata + Assert.True((bool)packageJson.forceRerunAfterDatabaseRestore); + Assert.Equal("CumulativeByMinor", (string)packageJson.upgradeStrategy); + + // deployment object metadata + Assert.Equal(1, (int)packageJson.deployment.manifestVersion); + Assert.Equal(150, (int)packageJson.deployment.minSqlCompatibility); + Assert.Equal("db/runtime", (string)packageJson.deployment.targetLayerDirectory); + Assert.False(string.IsNullOrEmpty((string)packageJson.deployment.buildDate)); + + // extended metadata + Assert.Equal("MyApp", (string)packageJson.deployment.metadata.ApplicationName); + Assert.Equal("11.1.0", (string)packageJson.deployment.metadata.ApplicationVersion); + Assert.Equal("customValue", (string)packageJson.deployment.metadata.customKey); + + // package demands + Assert.Equal(1, (int)packageJson.deployment.packageDemands.Count); + Assert.Equal("SQLServer", (string)packageJson.deployment.packageDemands[0].type); + Assert.Equal("14.0.1000.169", (string)packageJson.deployment.packageDemands[0].version); + Assert.Equal("Microsoft SQL Server 2017", (string)packageJson.deployment.packageDemands[0].description); + + // root package keyword + string keywords = packageJson.keywords.ToString(); + keywords.Should().Contain("cmf-deployment-package"); + keywords.Should().Contain("cmf-deployment-rootPackage"); + } + [Fact] public void RepositoriesConfig_JsonDeserialization_VariousUriFormats() {