Add opt-in analyze support for extensionless/unknown file types

DevSkim-DotNet/Microsoft.DevSkim.CLI/Commands/AnalyzeCommand.cs

@@ -100,7 +100,7 @@ public int Run()
                     if (IsGitPresent())
                     {
                         List<FileEntry> innerList = new List<FileEntry>();
-                        IEnumerable<string> files = Directory.EnumerateFiles(fullPath, "*.*", SearchOption.AllDirectories)
+                        IEnumerable<string> files = Directory.EnumerateFiles(fullPath, "*", SearchOption.AllDirectories)
                             .Where(fileName => !IsGitIgnored(fileName));
                         foreach (string? notIgnoredFileName in files)
                         {
@@ -119,7 +119,7 @@ public int Run()
                 else
                 {
                     List<FileEntry> innerList = new List<FileEntry>();
-                    IEnumerable<string> files = Directory.EnumerateFiles(fullPath, "*.*", SearchOption.AllDirectories);
+                    IEnumerable<string> files = Directory.EnumerateFiles(fullPath, "*", SearchOption.AllDirectories);
                     foreach (string file in files)
                     {
                         innerList.AddRange(FilePathToFileEntries(_opts, file, extractor, extractorOpts));
@@ -393,8 +393,9 @@ void parseFileEntry(FileEntry fileEntry)
             {
                 devSkimLanguages.FromFileNameOut(fileEntry.Name, out LanguageInfo languageInfo);
 
-                // Skip files written in unknown language
-                if (string.IsNullOrEmpty(languageInfo.Name))
+                // Skip files written in unknown language unless explicitly configured not to
+                bool isUnknownLanguage = string.IsNullOrEmpty(languageInfo.Name);
+                if (isUnknownLanguage && !_opts.AnalyzeUnknownFileTypes)
                 {
                     Interlocked.Increment(ref filesSkipped);
                 }
@@ -457,7 +458,7 @@ void parseFileEntry(FileEntry fileEntry)
                                     Filesize: fileText.Length,
                                     TextSample: _opts.SkipExcerpts ? string.Empty : issueText,
                                     Issue: issue,
-                                    Language: languageInfo.Name,
+                                    Language: isUnknownLanguage ? "unknown" : languageInfo.Name,
                                     Fixes: issue.Rule.Fixes?.Where(x => DevSkimRuleProcessor.IsFixable(issueText, x)).ToList());
 
                                 outputWriter.WriteIssue(record);
@@ -539,4 +540,4 @@ private ICollection<FileEntry> FilenameToFileEntryArray(string pathToFile)
             return Array.Empty<FileEntry>();
         }
     }
-}
+}

DevSkim-DotNet/Microsoft.DevSkim.CLI/Options/BaseAnalyzeCommandOptions.cs

@@ -76,4 +76,7 @@ public record BaseAnalyzeCommandOptions : LogOptions
 
     [Option("skip-excerpts", HelpText = "Set to skip gathering excerpts and samples to include in the report.", Default = false)]
     public bool SkipExcerpts { get; set; }
-}
+
+    [Option("analyze-unknown-file-types", HelpText = "Set to analyze files with unknown language/file types.", Default = false)]
+    public bool AnalyzeUnknownFileTypes { get; set; }
+}

DevSkim-DotNet/Microsoft.DevSkim.Tests/AnalyzeTest.cs

@@ -312,5 +312,69 @@ public void TestConfigXPathRulesApplyOnlyToConfigFiles()
                 .Where(r => r.RuleId == "DS450001" || r.RuleId == "DS450002" || r.RuleId == "DS450003");
             Assert.AreEqual(0, configJsonRuleMatches.Count(), "Config xpath rules should NOT match .config.json files");
         }
+
+        [DataTestMethod]
+        [DataRow(false, 0)]
+        [DataRow(true, 1)]
+        public void TestAnalyzeUnknownExtensionlessFiles(bool analyzeUnknownFileTypes, int expectedNumResults)
+        {
+            string testDirectory = Path.Combine(Path.GetTempPath(), $"{nameof(TestAnalyzeUnknownExtensionlessFiles)}_{Guid.NewGuid()}");
+            Directory.CreateDirectory(testDirectory);
+
+            try
+            {
+                string extensionlessFileName = Path.Combine(testDirectory, "passwd");
+                string outFileName = PathHelper.GetRandomTempFile("sarif");
+                string ruleFileName = PathHelper.GetRandomTempFile("json");
+
+                string passwdContent = @"root:x:0:0:root:/root:/bin/bash";
+                string ruleFileContent = @"[
+    {
+        ""name"": ""Detect /etc/passwd-like syntax"",
+        ""id"": ""DSTESTPASSWD"",
+        ""description"": ""Detects /etc/passwd-like syntax."",
+        ""severity"": ""BestPractice"",
+        ""confidence"": ""High"",
+        ""tags"": [""security"", ""passwd""],
+        ""applies_to_file_regex"": [""(^|[\\\\/])passwd$""],
+        ""patterns"": [
+            {
+                ""pattern"": ""^root:x:0:0:root:/root:/bin/bash$"",
+                ""type"": ""regex"",
+                ""modifiers"": [""m""],
+                ""scopes"": [""all""]
+            }
+        ]
+    }
+]";
+
+                File.WriteAllText(extensionlessFileName, passwdContent);
+                File.WriteAllText(ruleFileName, ruleFileContent);
+
+                AnalyzeCommandOptions opts = new AnalyzeCommandOptions()
+                {
+                    Path = testDirectory,
+                    OutputFile = outFileName,
+                    OutputFileFormat = "sarif",
+                    Rules = new[] { ruleFileName },
+                    IgnoreDefaultRules = true,
+                    AnalyzeUnknownFileTypes = analyzeUnknownFileTypes
+                };
+
+                int resultCode = new AnalyzeCommand(opts).Run();
+                Assert.AreEqual((int)ExitCode.Okay, resultCode);
+
+                SarifLog resultsFile = SarifLog.Load(outFileName);
+                Assert.AreEqual(1, resultsFile.Runs.Count);
+                Assert.AreEqual(expectedNumResults, resultsFile.Runs[0].Results?.Count ?? 0);
+            }
+            finally
+            {
+                if (Directory.Exists(testDirectory))
+                {
+                    Directory.Delete(testDirectory, true);
+                }
+            }
+        }
     }
-}
+}