From b96e6865765ca7d8d2f5eb81a69c9f3de68d569f Mon Sep 17 00:00:00 2001 From: Tom Harrington Date: Tue, 8 Feb 2022 14:56:55 -0700 Subject: [PATCH 1/8] Add Website option to ignore paths by name regex --- Sources/Publish/API/PublishingContext.swift | 3 +++ Sources/Publish/API/Website.swift | 11 +++++++++++ Sources/Publish/Internal/MarkdownFileHandler.swift | 3 +++ 3 files changed, 17 insertions(+) diff --git a/Sources/Publish/API/PublishingContext.swift b/Sources/Publish/API/PublishingContext.swift index 4dd14caa..39f478c9 100644 --- a/Sources/Publish/API/PublishingContext.swift +++ b/Sources/Publish/API/PublishingContext.swift @@ -293,6 +293,7 @@ internal extension PublishingContext { func copyFileToOutput(_ file: File, targetFolderPath: Path?) throws { + guard !site.shouldIgnore(name: file.name) else { return } try copyLocationToOutput( file, targetFolderPath: targetFolderPath, @@ -302,6 +303,7 @@ internal extension PublishingContext { func copyFolderToOutput(_ folder: Folder, targetFolderPath: Path?) throws { + guard !site.shouldIgnore(name: folder.name) else { return } try copyLocationToOutput( folder, targetFolderPath: targetFolderPath, @@ -349,6 +351,7 @@ private extension PublishingContext { targetFolderPath: Path?, errorReason: FileIOError.Reason ) throws { + guard !site.shouldIgnore(name: location.name) else { return } let targetFolder = try targetFolderPath.map { try createOutputFolder(at: $0) } diff --git a/Sources/Publish/API/Website.swift b/Sources/Publish/API/Website.swift index 75ae2b5a..78ca046d 100644 --- a/Sources/Publish/API/Website.swift +++ b/Sources/Publish/API/Website.swift @@ -40,6 +40,8 @@ public protocol Website { /// The configuration to use when generating tag HTML for the website. /// If this is `nil`, then no tag HTML will be generated. var tagHTMLConfig: TagHTMLConfiguration? { get } + /// File or folder names to exclude when adding Markdown files. Regular expressions may be used. + var ignoredPaths: [String]? { get } } // MARK: - Defaults @@ -154,4 +156,13 @@ public extension Website { func url(for location: Location) -> URL { url(for: location.path) } + + var ignoredPaths: [String]? { nil } + + func shouldIgnore(name: String) -> Bool { + guard let ignoredPaths = ignoredPaths else { return false } + return !ignoredPaths.filter({ + name.range(of: $0, options: .regularExpression) != nil + }).isEmpty + } } diff --git a/Sources/Publish/Internal/MarkdownFileHandler.swift b/Sources/Publish/Internal/MarkdownFileHandler.swift index 14dd36f9..9446a390 100644 --- a/Sources/Publish/Internal/MarkdownFileHandler.swift +++ b/Sources/Publish/Internal/MarkdownFileHandler.swift @@ -22,6 +22,7 @@ internal struct MarkdownFileHandler { } for subfolder in folder.subfolders { + guard !context.site.shouldIgnore(name: subfolder.name) else { return } guard let sectionID = Site.SectionID(rawValue: subfolder.name.lowercased()) else { try addPagesForMarkdownFiles( inFolder: subfolder, @@ -35,6 +36,7 @@ internal struct MarkdownFileHandler { } for file in subfolder.files.recursive { + guard !context.site.shouldIgnore(name: file.name) else { return } guard file.isMarkdown else { continue } if file.nameExcludingExtension == "index", file.parent == subfolder { @@ -86,6 +88,7 @@ private extension MarkdownFileHandler { factory: MarkdownContentFactory ) throws { for file in folder.files { + guard !context.site.shouldIgnore(name: file.name) else { return } guard file.isMarkdown else { continue } if file.nameExcludingExtension == "index", !recursively { From dd325f6880ad9b33e0bf6e5ac801175b1b4010b0 Mon Sep 17 00:00:00 2001 From: Tom Harrington Date: Tue, 15 Feb 2022 15:22:06 -0700 Subject: [PATCH 2/8] Fix a bug where one ignored file could cause a bunch of unrelated files to be ignored --- Sources/Publish/Internal/MarkdownFileHandler.swift | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Sources/Publish/Internal/MarkdownFileHandler.swift b/Sources/Publish/Internal/MarkdownFileHandler.swift index 9446a390..8b6850f2 100644 --- a/Sources/Publish/Internal/MarkdownFileHandler.swift +++ b/Sources/Publish/Internal/MarkdownFileHandler.swift @@ -22,7 +22,7 @@ internal struct MarkdownFileHandler { } for subfolder in folder.subfolders { - guard !context.site.shouldIgnore(name: subfolder.name) else { return } + guard !context.site.shouldIgnore(name: subfolder.name) else { continue } guard let sectionID = Site.SectionID(rawValue: subfolder.name.lowercased()) else { try addPagesForMarkdownFiles( inFolder: subfolder, @@ -36,7 +36,7 @@ internal struct MarkdownFileHandler { } for file in subfolder.files.recursive { - guard !context.site.shouldIgnore(name: file.name) else { return } + guard !context.site.shouldIgnore(name: file.name) else { continue } guard file.isMarkdown else { continue } if file.nameExcludingExtension == "index", file.parent == subfolder { @@ -88,7 +88,7 @@ private extension MarkdownFileHandler { factory: MarkdownContentFactory ) throws { for file in folder.files { - guard !context.site.shouldIgnore(name: file.name) else { return } + guard !context.site.shouldIgnore(name: file.name) else { continue } guard file.isMarkdown else { continue } if file.nameExcludingExtension == "index", !recursively { From 65734766aae65ab574b600ac09f734d87aaca81b Mon Sep 17 00:00:00 2001 From: Tom Harrington Date: Tue, 15 Feb 2022 15:34:45 -0700 Subject: [PATCH 3/8] Improve matching to avoid unexpected matches --- Sources/Publish/API/Website.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Sources/Publish/API/Website.swift b/Sources/Publish/API/Website.swift index 78ca046d..159858df 100644 --- a/Sources/Publish/API/Website.swift +++ b/Sources/Publish/API/Website.swift @@ -163,6 +163,8 @@ public extension Website { guard let ignoredPaths = ignoredPaths else { return false } return !ignoredPaths.filter({ name.range(of: $0, options: .regularExpression) != nil + // Add `^` and `$` to the ends of the pattern to avoid unexpected matches. Matching something like "foo" anywhere in a filename requires wildcards, e.g. ".*foo.*". Extra line start/end markers don't affect matching, so there's no need to check for them before trying the pattern. + name.range(of: "^\($0)$", options: .regularExpression) != nil }).isEmpty } } From e01e974b4df85f9be5ff376398f8f8ef48366d5b Mon Sep 17 00:00:00 2001 From: Tom Harrington Date: Tue, 15 Feb 2022 15:35:06 -0700 Subject: [PATCH 4/8] Add some info on how to use ignoredPaths and shouldIgnore --- Sources/Publish/API/Website.swift | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/Sources/Publish/API/Website.swift b/Sources/Publish/API/Website.swift index 159858df..c2348631 100644 --- a/Sources/Publish/API/Website.swift +++ b/Sources/Publish/API/Website.swift @@ -40,7 +40,10 @@ public protocol Website { /// The configuration to use when generating tag HTML for the website. /// If this is `nil`, then no tag HTML will be generated. var tagHTMLConfig: TagHTMLConfiguration? { get } - /// File or folder names to exclude when adding Markdown files. Regular expressions may be used. + /// File or folder names to exclude when publishing the site. Regular expressions may be used. + /// - Bare strings with no regexp-related characters will be matched exactly. For example `"templates"` will ignore anything named `templates` but not something named `templates1` or `my templates`, etc. + /// - Wildcards are allowed and follow usual regular expression meanings. For example, `templates.*` means to ignore anything that starts with `templates` and includes zero or more characters after `templates`. + /// - It's not necessary to use `^` or `$` to indicate the start or end of a regular expression, but it's not harmful either. var ignoredPaths: [String]? { get } } @@ -157,12 +160,19 @@ public extension Website { url(for: location.path) } + /// Make `ignoredPaths` optional. var ignoredPaths: [String]? { nil } - + + /// Should the site ignore a file or folder with a specific name? + /// - Parameter name: Name of a file or folder, commonly `file.name` or `folder.name`. + /// - Returns: True if the web site should ignore the file/folder name + /// + /// Sites can indicate file/folder names to ignore using the `ignoredPaths` property. See that property for a discussion of how to use it. + /// + /// If a folder is ignored, everything in that folder will be ignored, regardless of whether the folder contents match anything in `ignoredPaths`. func shouldIgnore(name: String) -> Bool { guard let ignoredPaths = ignoredPaths else { return false } return !ignoredPaths.filter({ - name.range(of: $0, options: .regularExpression) != nil // Add `^` and `$` to the ends of the pattern to avoid unexpected matches. Matching something like "foo" anywhere in a filename requires wildcards, e.g. ".*foo.*". Extra line start/end markers don't affect matching, so there's no need to check for them before trying the pattern. name.range(of: "^\($0)$", options: .regularExpression) != nil }).isEmpty From b4377a44f89dac2302fbadd77a580b5aa60a8854 Mon Sep 17 00:00:00 2001 From: Tom Harrington Date: Tue, 15 Feb 2022 15:35:22 -0700 Subject: [PATCH 5/8] Add some tests for shouldIgnore --- Tests/PublishTests/Tests/WebsiteTests.swift | 35 +++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/Tests/PublishTests/Tests/WebsiteTests.swift b/Tests/PublishTests/Tests/WebsiteTests.swift index 1ab15744..1e0872a3 100644 --- a/Tests/PublishTests/Tests/WebsiteTests.swift +++ b/Tests/PublishTests/Tests/WebsiteTests.swift @@ -81,4 +81,39 @@ final class WebsiteTests: PublishTestCase { URL(string: "https://swiftbysundell.com/mypage") ) } + + func testIgnorePatterns() { + XCTAssertTrue(website.shouldIgnore(name: "templates")) + XCTAssertFalse(website.shouldIgnore(name: "templates1")) + XCTAssertFalse(website.shouldIgnore(name: "@templates")) + + XCTAssertTrue(website.shouldIgnore(name: "skip-this-file.md")) + XCTAssertTrue(website.shouldIgnore(name: "skip-this-file-too.png")) + XCTAssertTrue(website.shouldIgnore(name: "skip-this-file")) + XCTAssertFalse(website.shouldIgnore(name: "dont-skip-this-file")) + + XCTAssertTrue(website.shouldIgnore(name: "notes")) + XCTAssertFalse(website.shouldIgnore(name: "notes1")) + + XCTAssertTrue(website.shouldIgnore(name: "batter")) + XCTAssertTrue(website.shouldIgnore(name: "butter")) + XCTAssertFalse(website.shouldIgnore(name: "butter1")) + + XCTAssertTrue(website.shouldIgnore(name: "lisp")) + XCTAssertTrue(website.shouldIgnore(name: "lip")) + XCTAssertTrue(website.shouldIgnore(name: "lp")) + XCTAssertTrue(website.shouldIgnore(name: "l1234567890p")) + XCTAssertFalse(website.shouldIgnore(name: "lisp-too")) + } } + +private extension Website { + var ignoredPaths: [String]? { [ + "templates", // Should only match the exact string + "skip-this-file.*", // Should match anyhing starting with "skip-this-file" and 0 or more chars after that + "^notes$", // Should match "notes" exactly and nothing else. Included because `shouldIgnore` adds a "^" and "$", which should not be affected by including those in the pattern. + "b.tter", + "l.*p" + ] } +} + From 7ba798ce3d5b2f73bb14d81eb10343566e069839 Mon Sep 17 00:00:00 2001 From: Tom Harrington Date: Tue, 15 Feb 2022 16:22:07 -0700 Subject: [PATCH 6/8] Access control fix for test --- Tests/PublishTests/Tests/WebsiteTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/PublishTests/Tests/WebsiteTests.swift b/Tests/PublishTests/Tests/WebsiteTests.swift index 1e0872a3..c027599a 100644 --- a/Tests/PublishTests/Tests/WebsiteTests.swift +++ b/Tests/PublishTests/Tests/WebsiteTests.swift @@ -107,7 +107,7 @@ final class WebsiteTests: PublishTestCase { } } -private extension Website { +extension Website { var ignoredPaths: [String]? { [ "templates", // Should only match the exact string "skip-this-file.*", // Should match anyhing starting with "skip-this-file" and 0 or more chars after that From 15d7f11dc004762f82338c8105f970feccfd2c9e Mon Sep 17 00:00:00 2001 From: Tom Harrington Date: Fri, 25 Feb 2022 12:31:53 -0700 Subject: [PATCH 7/8] Add ignoring File/Folder instances directly --- Sources/Publish/API/PublishingContext.swift | 6 +++--- Sources/Publish/API/Website.swift | 21 ++++++++++++++++++- .../Internal/MarkdownFileHandler.swift | 6 +++--- Tests/PublishTests/Tests/WebsiteTests.swift | 19 +++++++++++++++-- 4 files changed, 43 insertions(+), 9 deletions(-) diff --git a/Sources/Publish/API/PublishingContext.swift b/Sources/Publish/API/PublishingContext.swift index 39f478c9..5b681071 100644 --- a/Sources/Publish/API/PublishingContext.swift +++ b/Sources/Publish/API/PublishingContext.swift @@ -293,7 +293,7 @@ internal extension PublishingContext { func copyFileToOutput(_ file: File, targetFolderPath: Path?) throws { - guard !site.shouldIgnore(name: file.name) else { return } + guard !site.shouldIgnore(file) else { return } try copyLocationToOutput( file, targetFolderPath: targetFolderPath, @@ -303,7 +303,7 @@ internal extension PublishingContext { func copyFolderToOutput(_ folder: Folder, targetFolderPath: Path?) throws { - guard !site.shouldIgnore(name: folder.name) else { return } + guard !site.shouldIgnore(folder) else { return } try copyLocationToOutput( folder, targetFolderPath: targetFolderPath, @@ -351,7 +351,7 @@ private extension PublishingContext { targetFolderPath: Path?, errorReason: FileIOError.Reason ) throws { - guard !site.shouldIgnore(name: location.name) else { return } + guard !site.shouldIgnore(location) else { return } let targetFolder = try targetFolderPath.map { try createOutputFolder(at: $0) } diff --git a/Sources/Publish/API/Website.swift b/Sources/Publish/API/Website.swift index c2348631..9fa06df8 100644 --- a/Sources/Publish/API/Website.swift +++ b/Sources/Publish/API/Website.swift @@ -6,6 +6,7 @@ import Foundation import Plot +import Files /// Protocol that all `Website.SectionID` implementations must conform to. public protocol WebsiteSectionID: Decodable, Hashable, CaseIterable, RawRepresentable where RawValue == String {} @@ -44,6 +45,7 @@ public protocol Website { /// - Bare strings with no regexp-related characters will be matched exactly. For example `"templates"` will ignore anything named `templates` but not something named `templates1` or `my templates`, etc. /// - Wildcards are allowed and follow usual regular expression meanings. For example, `templates.*` means to ignore anything that starts with `templates` and includes zero or more characters after `templates`. /// - It's not necessary to use `^` or `$` to indicate the start or end of a regular expression, but it's not harmful either. + /// If a folder is ignored, everything in that folder will be ignored, regardless of whether the folder contents match anything in `ignoredPaths`. var ignoredPaths: [String]? { get } } @@ -169,7 +171,7 @@ public extension Website { /// /// Sites can indicate file/folder names to ignore using the `ignoredPaths` property. See that property for a discussion of how to use it. /// - /// If a folder is ignored, everything in that folder will be ignored, regardless of whether the folder contents match anything in `ignoredPaths`. + /// This function checks only strings and should be called with a file or folder name. func shouldIgnore(name: String) -> Bool { guard let ignoredPaths = ignoredPaths else { return false } return !ignoredPaths.filter({ @@ -177,4 +179,21 @@ public extension Website { name.range(of: "^\($0)$", options: .regularExpression) != nil }).isEmpty } + + /// Should the site ignore a `File` or `Folder`? + /// - Returns: True if the site should ignore the `File`/`Folder`. + /// + /// Sites can indicate file/folder names to ignore using the `ignoredPaths` property. See that property for a discussion of how to use it. + /// + /// *This function checks all path components for the location.* A file or folder that doesn't match the ignore list could still be contained in a folder that does match. This function deals with this by checking all path components against the `ignoredPaths` list and returning true if any matches are found. If you want to check a file or folder's name without checking the entire path, use `shouldIgnore(name:)` + func shouldIgnore(_ location: T) -> Bool { + location.pathComponents.reduce(false) { (_, component) in shouldIgnore(name: component) } + } +} + +extension Files.Location { + // Used in `shouldIgnore(_ location)` + var pathComponents: [String] { + path.split(separator: "/").map { String($0) } + } } diff --git a/Sources/Publish/Internal/MarkdownFileHandler.swift b/Sources/Publish/Internal/MarkdownFileHandler.swift index 8b6850f2..3155ef20 100644 --- a/Sources/Publish/Internal/MarkdownFileHandler.swift +++ b/Sources/Publish/Internal/MarkdownFileHandler.swift @@ -22,7 +22,7 @@ internal struct MarkdownFileHandler { } for subfolder in folder.subfolders { - guard !context.site.shouldIgnore(name: subfolder.name) else { continue } + guard !context.site.shouldIgnore(subfolder) else { continue } guard let sectionID = Site.SectionID(rawValue: subfolder.name.lowercased()) else { try addPagesForMarkdownFiles( inFolder: subfolder, @@ -36,7 +36,7 @@ internal struct MarkdownFileHandler { } for file in subfolder.files.recursive { - guard !context.site.shouldIgnore(name: file.name) else { continue } + guard !context.site.shouldIgnore(file) else { continue } guard file.isMarkdown else { continue } if file.nameExcludingExtension == "index", file.parent == subfolder { @@ -88,7 +88,7 @@ private extension MarkdownFileHandler { factory: MarkdownContentFactory ) throws { for file in folder.files { - guard !context.site.shouldIgnore(name: file.name) else { continue } + guard !context.site.shouldIgnore(file) else { continue } guard file.isMarkdown else { continue } if file.nameExcludingExtension == "index", !recursively { diff --git a/Tests/PublishTests/Tests/WebsiteTests.swift b/Tests/PublishTests/Tests/WebsiteTests.swift index c027599a..a00e009c 100644 --- a/Tests/PublishTests/Tests/WebsiteTests.swift +++ b/Tests/PublishTests/Tests/WebsiteTests.swift @@ -6,6 +6,7 @@ import XCTest import Publish +import Files final class WebsiteTests: PublishTestCase { private var website: WebsiteStub.WithoutItemMetadata! @@ -82,7 +83,7 @@ final class WebsiteTests: PublishTestCase { ) } - func testIgnorePatterns() { + func testIgnoreStrings() { XCTAssertTrue(website.shouldIgnore(name: "templates")) XCTAssertFalse(website.shouldIgnore(name: "templates1")) XCTAssertFalse(website.shouldIgnore(name: "@templates")) @@ -105,6 +106,18 @@ final class WebsiteTests: PublishTestCase { XCTAssertTrue(website.shouldIgnore(name: "l1234567890p")) XCTAssertFalse(website.shouldIgnore(name: "lisp-too")) } + + func testIgnoreFiles() { + // Set up a couple of real files to test ignore-matching on a File, since they require existing files. + let folder: Folder! = try! Folder.temporary.createSubfolderIfNeeded(withName: ".publishTest") + let includeFile = try! folder.createFile(named: "include.txt") + XCTAssertFalse(website.shouldIgnore(includeFile)) + + let ignoreFile = try! folder.createFile(named: "ignore.txt") + XCTAssertTrue(website.shouldIgnore(ignoreFile)) + + try? folder.delete() + } } extension Website { @@ -113,7 +126,9 @@ extension Website { "skip-this-file.*", // Should match anyhing starting with "skip-this-file" and 0 or more chars after that "^notes$", // Should match "notes" exactly and nothing else. Included because `shouldIgnore` adds a "^" and "$", which should not be affected by including those in the pattern. "b.tter", - "l.*p" + "l.*p", +// (#file as NSString).lastPathComponent + "ignore.txt" ] } } From 798ffa1273fd1721c2bf5ae80d1cf5cbcf5e016c Mon Sep 17 00:00:00 2001 From: Tom Harrington Date: Mon, 11 Apr 2022 14:58:39 -0600 Subject: [PATCH 8/8] Fix bug where folder ignore status could be overwritten by a file in the folder --- Sources/Publish/API/Website.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Publish/API/Website.swift b/Sources/Publish/API/Website.swift index 9fa06df8..a278f7e9 100644 --- a/Sources/Publish/API/Website.swift +++ b/Sources/Publish/API/Website.swift @@ -187,7 +187,7 @@ public extension Website { /// /// *This function checks all path components for the location.* A file or folder that doesn't match the ignore list could still be contained in a folder that does match. This function deals with this by checking all path components against the `ignoredPaths` list and returning true if any matches are found. If you want to check a file or folder's name without checking the entire path, use `shouldIgnore(name:)` func shouldIgnore(_ location: T) -> Bool { - location.pathComponents.reduce(false) { (_, component) in shouldIgnore(name: component) } + !location.pathComponents.filter({ shouldIgnore(name:$0) }).isEmpty } }