diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a761305aafd..c39d32d6306 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -102,20 +102,6 @@ updates: - miloush - mcdurdin - - package-ecosystem: "npm" - directory: "/docs/charts/keyboards" - schedule: - interval: "monthly" - commit-message: - include: scope - prefix: "CLDR-19055 kbd:" - assignees: - - srl295 - reviewers: - - srl295 - - miloush - - mcdurdin - # Maintain dependencies for Maven - package-ecosystem: "maven" directory: "/tools" diff --git a/.github/workflows/keyboard.yml b/.github/workflows/keyboard.yml index 42bc2f52854..462cd2d2eb3 100644 --- a/.github/workflows/keyboard.yml +++ b/.github/workflows/keyboard.yml @@ -39,5 +39,3 @@ jobs: run: kmc --error-reporting build keyboards/3.0/*.xml - name: Check ABNF run: 'cd tools/scripts/keyboard-abnf-tests && npm ci && npm t' - - name: Run Kbd Charts - run: 'cd docs/charts/keyboards && npm ci && npm run build' diff --git a/.github/workflows/spec.yml b/.github/workflows/spec.yml index 2aefe3ccc74..c6d3f16a381 100644 --- a/.github/workflows/spec.yml +++ b/.github/workflows/spec.yml @@ -52,8 +52,6 @@ jobs: - name: Run TR archiver # Note: will update ToC if out of date run: "cd tools/scripts/tr-archive/ && npm ci && npm run build" - - name: Run Kbd Charts - run: "cd docs/charts/keyboards && npm ci && npm run build" - name: Upload tr35.zip uses: actions/upload-artifact@v7 with: diff --git a/docs/charts/keyboards/.gitignore b/docs/charts/keyboards/.gitignore deleted file mode 100644 index ffb5012011c..00000000000 --- a/docs/charts/keyboards/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -/node_modules -/static/data -/node_modules -/node -/target diff --git a/docs/charts/keyboards/README.md b/docs/charts/keyboards/README.md deleted file mode 100644 index bd33fd9e7d0..00000000000 --- a/docs/charts/keyboards/README.md +++ /dev/null @@ -1,30 +0,0 @@ -# Keyboard Charts - -## What are these - -The Keyboard Charts are now built as client-side JavaScript tables. - -## To Build from the command line - -- install current LTS version, then in this directory: -- `npm i` -- `npm run build` - -## To build from Maven - -Run this from the command line in the top level directory: - -- `mvn --file=tools/pom.xml -pl :cldr-keyboard-charts integration-test` - -## To build from Eclipse - -- Right-click on the `pom.xml` in `docs/charts/keyboards` (it will show as "cldr-keyboard-charts") -- Choose **Run As... - Maven Build** -- Change the **goal** to `integration-test` -- Choose Run - -## Trying them out - -- `npm run serve` will serve the charts locally on -- Or view the [`index.html`](./index.html) file located in the same directory as this README - diff --git a/docs/charts/keyboards/build.mjs b/docs/charts/keyboards/build.mjs deleted file mode 100644 index 73c4f294745..00000000000 --- a/docs/charts/keyboards/build.mjs +++ /dev/null @@ -1,102 +0,0 @@ -// do the XML parsing and fs access in a build step - -import { promises as fs } from "node:fs"; -import * as path from "node:path"; -import { XMLParser } from "fast-xml-parser"; - -const KEYBOARD_PATH = "../../../keyboards/3.0"; -const IMPORT_PATH = "../../../keyboards/import"; -const DATA_PATH = "static/data"; - -async function xmlList(basepath) { - const dir = await fs.opendir(basepath); - const xmls = []; - for await (const ent of dir) { - if (!ent.isFile() || !/\.xml$/.test(ent.name)) { - continue; - } - xmls.push(ent.name); - } - return xmls; -} - -/** - * List of elements that are always arrays - */ -const alwaysArray = [ - "keyboard3.transforms", - "keyboard3.transforms.transformGroup", - "keyboard3.transforms.transformGroup.transform", -]; - -/** - * Loading helper for isArray - * @param name - * @param jpath - * @param isLeafNode - * @param isAttribute - * @returns - */ -// eslint-disable-next-line @typescript-eslint/no-unused-vars -const isArray = (name, jpath, isLeafNode, isAttribute) => { - if (alwaysArray.indexOf(jpath) !== -1) return true; - return false; -}; - -/** - * Do the XML Transform given raw XML source - * @param xml XML source for transforms. entire keyboard file. - * @param source source text - * @returns target text - */ -export function parseXml(xml) { - const parser = new XMLParser({ - ignoreAttributes: false, - isArray, - }); - const j = parser.parse(xml); - return j; -} - -async function readFile(path) { - return fs.readFile(path, "utf-8"); -} - -async function main() { - const xmls = await xmlList(KEYBOARD_PATH); - const keyboards = await packXmls(KEYBOARD_PATH, xmls); - const importFiles = await xmlList(IMPORT_PATH); - const imports = await packXmls(IMPORT_PATH, importFiles); - - const allData = { - keyboards, - imports, - }; - - const outPath = path.join(DATA_PATH, "keyboard-data.json"); - const outJsPath = path.join(DATA_PATH, "keyboard-data.js"); - await fs.mkdir(DATA_PATH, { recursive: true }); - const json = JSON.stringify(allData, null, " "); // indent, in case we need to read it - await fs.writeFile(outPath, json, "utf-8"); - await fs.writeFile(outJsPath, `const _KeyboardData = \n` + json); - return { xmls, importFiles, outPath, outJsPath }; -} - -main().then( - (done) => console.dir({ done }), - (err) => { - console.error(err); - process.exitCode = 1; - } -); - -async function packXmls(basepath, xmls) { - const allData = {}; - for (const fn of xmls) { - const fp = path.join(basepath, fn); - const data = await readFile(fp); - const parsed = parseXml(data); - allData[fn] = parsed; - } - return allData; -} diff --git a/docs/charts/keyboards/index.html b/docs/charts/keyboards/index.html deleted file mode 100644 index bcb7c7f3bbf..00000000000 --- a/docs/charts/keyboards/index.html +++ /dev/null @@ -1,2 +0,0 @@ -

You're almost there

-Click here for the keyboard charts diff --git a/docs/charts/keyboards/package-lock.json b/docs/charts/keyboards/package-lock.json deleted file mode 100644 index 3481abb9301..00000000000 --- a/docs/charts/keyboards/package-lock.json +++ /dev/null @@ -1,108 +0,0 @@ -{ - "name": "@unicode-org/keyboard-charts", - "version": "1.0.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "@unicode-org/keyboard-charts", - "version": "1.0.0", - "hasInstallScript": true, - "license": "Unicode-3.0", - "dependencies": { - "fast-xml-parser": "^5.7.2" - } - }, - "node_modules/@nodable/entities": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-2.1.0.tgz", - "integrity": "sha512-nyT7T3nbMyBI/lvr6L5TyWbFJAI9FTgVRakNoBqCD+PmID8DzFrrNdLLtHMwMszOtqZa8PAOV24ZqDnQrhQINA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/nodable" - } - ], - "license": "MIT" - }, - "node_modules/fast-xml-builder": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.2.0.tgz", - "integrity": "sha512-00aAWieqff+ZJhsXA4g1g7M8k+7AYoMUUHF+/zFb5U6Uv/P0Vl4QZo84/IcufzYalLuEj9928bXN9PbbFzMF0Q==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT", - "dependencies": { - "path-expression-matcher": "^1.5.0", - "xml-naming": "^0.1.0" - } - }, - "node_modules/fast-xml-parser": { - "version": "5.7.2", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.7.2.tgz", - "integrity": "sha512-P7oW7tLbYnhOLQk/Gv7cZgzgMPP/XN03K02/Jy6Y/NHzyIAIpxuZIM/YqAkfiXFPxA2CTm7NtCijK9EDu09u2w==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT", - "dependencies": { - "@nodable/entities": "^2.1.0", - "fast-xml-builder": "^1.1.5", - "path-expression-matcher": "^1.5.0", - "strnum": "^2.2.3" - }, - "bin": { - "fxparser": "src/cli/cli.js" - } - }, - "node_modules/path-expression-matcher": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.5.0.tgz", - "integrity": "sha512-cbrerZV+6rvdQrrD+iGMcZFEiiSrbv9Tfdkvnusy6y0x0GKBXREFg/Y65GhIfm0tnLntThhzCnfKwp1WRjeCyQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT", - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/strnum": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.2.3.tgz", - "integrity": "sha512-oKx6RUCuHfT3oyVjtnrmn19H1SiCqgJSg+54XqURKp5aCMbrXrhLjRN9TjuwMjiYstZ0MzDrHqkGZ5dFTKd+zg==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT" - }, - "node_modules/xml-naming": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.1.0.tgz", - "integrity": "sha512-k8KO9hrMyNk6tUWqUfkTEZbezRRpONVOzUTnc97VnCvyj6Tf9lyUR9EDAIeiVLv56jsMcoXEwjW8Kv5yPY52lw==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT", - "engines": { - "node": ">=16.0.0" - } - } - } -} diff --git a/docs/charts/keyboards/package.json b/docs/charts/keyboards/package.json deleted file mode 100644 index 8e6dc461109..00000000000 --- a/docs/charts/keyboards/package.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "name": "@unicode-org/keyboard-charts", - "version": "1.0.0", - "description": "Keyboard Charts app", - "main": "index.js", - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1", - "serve": "npx serve static", - "build": "node build.mjs", - "lint": "echo no linter yet", - "postinstall": "node build.mjs" - }, - "keywords": [], - "author": "Steven R. Loomis ", - "license": "Unicode-3.0", - "bugs": { - "url": "https://github.com/unicode-org/cldr/issues" - }, - "homepage": "https://github.com/unicode-org/cldr#readme", - "private": true, - "dependencies": { - "fast-xml-parser": "^5.7.2" - } -} diff --git a/docs/charts/keyboards/pom.xml b/docs/charts/keyboards/pom.xml deleted file mode 100644 index 37ea3fef8fc..00000000000 --- a/docs/charts/keyboards/pom.xml +++ /dev/null @@ -1,112 +0,0 @@ - - - - 4.0.0 - - cldr-keyboard-charts - pom - CLDR Keyboard Charts - - https://unicode.org/cldr - - - scm:git:https://github.com/unicode-org/cldr.git - - - - - org.unicode.cldr - cldr-all - 49.0-SNAPSHOT - ../../../tools/pom.xml - - - - - - - maven-antrun-plugin - - - initialize - - - node.version= - - - - - - - - - run - - - - - - org.codehaus.mojo - properties-maven-plugin - - - initialize - - read-project-properties - - - - ${project.basedir}/target/node-version.properties - - - - - - - com.github.eirslett - frontend-maven-plugin - - - install node and npm - - install-node-and-npm - - integration-test - - - npm install - - npm - - integration-test - - - npm run build - - npm - - - run build - - integration-test - - - - ${node.version} - - ${project.basedir} - - - - - - - - - build-keyboard-charts - - - - diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateKeyboardCharts.java b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateKeyboardCharts.java index e4b0e5de04a..505dfb816e2 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateKeyboardCharts.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/GenerateKeyboardCharts.java @@ -1,18 +1,41 @@ package org.unicode.cldr.tool; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; import java.io.File; import java.io.IOException; +import java.nio.file.FileVisitOption; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; +import java.util.List; +import java.util.stream.Collectors; import org.unicode.cldr.util.CLDRPaths; +import org.unicode.cldr.util.TempPrintWriter; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NamedNodeMap; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; public class GenerateKeyboardCharts { + static final Gson gson = new GsonBuilder().setPrettyPrinting().create(); static final String SUBDIR = "keyboards"; static IOException copyErr = null; public static void main(String args[]) throws IOException { + final boolean makeSymlink = (args.length > 0 && args[0].equals("-l")); + if (makeSymlink) { + System.err.println("-l: making symlinks"); + } else { + System.err.println("(use -l to symlink)"); + } final File mainDir = new File(CLDRPaths.CHART_DIRECTORY); if (mainDir.mkdirs()) { System.err.println("Created: " + mainDir); @@ -20,31 +43,22 @@ public static void main(String args[]) throws IOException { if (!mainDir.isDirectory()) { throw new IOException("Main dir doesn't exist: " + mainDir); } - final File kbdDir = new File(CLDRPaths.BASE_DIRECTORY, "docs/charts/" + SUBDIR); + final File kbdDir = + new File( + CLDRPaths.BASE_DIRECTORY, + "tools/cldr-code/src/main/resources/org/unicode/cldr/util/data/keyboard-charts"); if (!kbdDir.exists()) { throw new IOException("Keyboards root dir doesn't exist: " + kbdDir); } - final File kbdStatic = new File(kbdDir, "static"); - final File kbdStaticData = new File(kbdDir, "static/data"); - if (!kbdStaticData.exists()) { - System.err.println( - "ERROR: " + kbdStaticData + " does not exist. Keyboard charts weren't run."); - System.err.println("See " + new File(kbdDir, "README.md") + " for help."); - return; - } - final File staticTarg = new File(mainDir, SUBDIR + "/static"); - final File staticDataTarg = new File(mainDir, SUBDIR + "/static/data"); + final File staticTarg = new File(mainDir, SUBDIR + "/"); + final File staticDataTarg = new File(mainDir, SUBDIR + "/data"); if (staticDataTarg.mkdirs()) { System.err.println("Created: " + staticDataTarg); } - System.out.println("Copying: " + kbdStatic + " to " + staticTarg); - - Files.copy( - new File(kbdDir, "index.html").toPath(), - new File(mainDir, SUBDIR + "/index.html").toPath(), - StandardCopyOption.REPLACE_EXISTING); - final String kbdStaticPrefix = kbdStatic.getAbsolutePath(); - Files.walk(kbdStatic.toPath()) + System.out.println("Copying: " + kbdDir + " to " + staticTarg); + + final String kbdStaticPrefix = kbdDir.getAbsolutePath(); + Files.walk(kbdDir.toPath()) .forEach( path -> { if (!path.toFile().isFile()) return; @@ -59,7 +73,17 @@ public static void main(String args[]) throws IOException { try { final Path out = new File(staticTarg, rel).toPath(); System.out.println(" -> " + out); - Files.copy(path, out, StandardCopyOption.REPLACE_EXISTING); + if (!makeSymlink) { + if (Files.isSymbolicLink(out)) { + Files.delete(out); + } + Files.copy(path, out, StandardCopyOption.REPLACE_EXISTING); + } else { + if (out.toFile().isFile()) { + out.toFile().delete(); + } + Files.createSymbolicLink(out, path); + } } catch (IOException e) { e.printStackTrace(); System.err.println("Error copying " + path); @@ -70,5 +94,118 @@ public static void main(String args[]) throws IOException { }); // rethrow any error if (copyErr != null) throw copyErr; + + // now, generate data + final Path keyboards3Dir = Path.of(CLDRPaths.KEYBOARDS_3_DIRECTORY); + System.err.println("Looking for keyboards in " + keyboards3Dir); + final List keyboardFiles = + Files.walk(keyboards3Dir, 1, FileVisitOption.FOLLOW_LINKS) + .filter( + p -> + p.toFile().isFile() + && p.toString().endsWith(".xml") + && !p.toString() + .endsWith( + "-test.xml")) // exclude test files + .collect(Collectors.toList()); + keyboardFiles.forEach(p -> System.err.println("- " + p.getFileName())); + + // calculate data + JsonObject root = new JsonObject(); + + JsonObject keyboards = generateKeyboards(keyboardFiles); + + root.add("keyboards", keyboards); + // write the data + final File outData = new File(staticDataTarg, "keyboard-data.json"); + + try (TempPrintWriter pw = new TempPrintWriter(outData)) { + pw.noDiff(); + pw.append(gson.toJson(root)); + } finally { + System.err.println("Wrote: " + outData.toString()); + } + + // copy to .js because of loading issues + final File outJs = new File(staticDataTarg, "keyboard-data.js"); + try (TempPrintWriter pw = new TempPrintWriter(outJs)) { + pw.noDiff(); + pw.println("const _KeyboardData = "); + pw.append(gson.toJson(root)); + pw.println(";"); + } finally { + System.err.println("Wrote: " + outData.toString()); + } + } + + private static JsonObject generateKeyboards(List keyboardFiles) { + final JsonObject keyboards = new JsonObject(); + + for (final Path p : keyboardFiles) { + try { + keyboards.add(p.getFileName().toString(), generateKeyboard(p)); + } catch (Throwable t) { + throw new RuntimeException("While processing " + p, t); + } + } + + return keyboards; + } + + private static JsonElement generateKeyboard(Path p) throws SAXException, IOException { + final InputSource is = KeyboardFlatten.getInputSource(p.toString()); + Document doc = KeyboardFlatten.flattenDoc(is, p.toString()); + JsonObject o = new JsonObject(); + + return appendObject(o, p, doc, doc.getDocumentElement()); + } + + private static JsonObject appendObject(JsonObject o, Path p, Document doc, Element e) { + JsonObject oo = new JsonObject(); + + NamedNodeMap attrs = e.getAttributes(); + appendAttrs(oo, p, doc, e, attrs); + + NodeList childNodes = e.getChildNodes(); + appendChildren(oo, p, doc, e, childNodes); + + String nodeName = e.getNodeName(); + if (o.has(nodeName)) { + JsonElement existing = o.get(nodeName); + if (existing.isJsonArray()) { + // add to array + existing.getAsJsonArray().add(oo); + } else { + // convert obj to array + o.remove(nodeName); + JsonArray a = new JsonArray(); + a.add(existing); + a.add(oo); + o.add(nodeName, a); + } + } else { + o.add(nodeName, oo); + } + return o; + } + + private static void appendChildren( + JsonObject o, Path p, Document doc, Element e, NodeList childNodes) { + if (childNodes == null) return; + for (int i = 0; i < childNodes.getLength(); i++) { + final Node n = childNodes.item(i); + if (n.getNodeType() == Node.ELEMENT_NODE) { + appendObject(o, p, doc, (Element) n); + } + } + } + + private static void appendAttrs( + JsonObject oo, Path p, Document doc, Element e, NamedNodeMap attrs) { + if (attrs == null) return; + for (int i = 0; i < attrs.getLength(); i++) { + final Node attr = attrs.item(i); + oo.addProperty("@_" + attr.getNodeName(), attr.getNodeValue()); + } } } diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/KeyboardFlatten.java b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/KeyboardFlatten.java index f93921083dd..54adcea9252 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/tool/KeyboardFlatten.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/tool/KeyboardFlatten.java @@ -1,8 +1,10 @@ package org.unicode.cldr.tool; import java.io.File; +import java.io.IOException; import java.io.OutputStream; -import java.net.MalformedURLException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.transform.TransformerConfigurationException; @@ -14,6 +16,8 @@ import javax.xml.validation.Schema; import javax.xml.validation.SchemaFactory; import org.unicode.cldr.util.CLDRConfig; +import org.unicode.cldr.util.CLDRFile; +import org.unicode.cldr.util.DoctypeXmlStreamWrapper; import org.unicode.cldr.util.DtdType; import org.unicode.cldr.util.PathUtilities; import org.unicode.cldr.util.XMLValidator; @@ -30,17 +34,23 @@ /** Read a Keyboard and write it out with no import statements */ public class KeyboardFlatten { public static void flatten(String path, OutputStream stream) - throws MalformedURLException, - SAXException, + throws SAXException, TransformerConfigurationException, TransformerException, - TransformerFactoryConfigurationError { - final String filename = PathUtilities.getNormalizedPathString(path); + TransformerFactoryConfigurationError, + IOException { + String filename = PathUtilities.getNormalizedPathString(path); + InputSource inputSource = getInputSource(filename); + flatten(inputSource, filename, stream); + } + + public static InputSource getInputSource(String filename) throws IOException { // Force filerefs to be URI's if needed: note this is independent of any // other files - String docURI; - docURI = XMLValidator.filenameToURL(filename); - flatten(new InputSource(docURI), filename, stream); + String docURI = XMLValidator.filenameToURL(filename); + InputSource inputSource = new InputSource(docURI); + inputSource = DoctypeXmlStreamWrapper.wrap(inputSource); + return inputSource; } public static void flatten(InputSource inputSource, String filename, OutputStream stream) @@ -48,7 +58,15 @@ public static void flatten(InputSource inputSource, String filename, OutputStrea TransformerConfigurationException, TransformerException, TransformerFactoryConfigurationError, - MalformedURLException { + IOException { + Document doc = flattenDoc(inputSource, filename); + + // Write out + write(doc, stream); + } + + public static Document flattenDoc(InputSource inputSource, String filename) + throws SAXException, IOException { final DocumentBuilderFactory dfactory = getKeyboardDocFactory(); final ErrorHandler nullHandler = getNullHandler(filename); // Parse @@ -56,14 +74,13 @@ public static void flatten(InputSource inputSource, String filename, OutputStrea // do the flatten flattenDoc(dfactory, doc); - - // Write out - write(doc, stream); + return doc; } private static void flattenDoc(final DocumentBuilderFactory dfactory, Document doc) - throws MalformedURLException { - // Now, flatten it + throws IOException { + + // apply explicit imports NodeList imports = doc.getElementsByTagName("import"); if (imports.getLength() == 0) { @@ -80,23 +97,36 @@ private static void flattenDoc(final DocumentBuilderFactory dfactory, Document d // item is no longer in list } } + + // Now, add implied keys + NodeList keyElements = doc.getElementsByTagName("keys"); + if (keyElements.getLength() == 1) { + // Now, apply implied imports + // applyImportItem(final DocumentBuilderFactory dfactory, Document doc, Node item, final + // String path, final String subpath) + applyImportItem( + dfactory, + doc, + keyElements.item(0).getFirstChild(), + CLDRFile.GEN_VERSION + "/keys-Latn-implied.xml", + "keys-Latn-implied.xml"); + } else { + System.err.println("Not 1 elements but " + keyElements.getLength()); + } } + private static final Pattern CLDR_PREFIX_PATTERN = Pattern.compile("^([0-9][0-9])/(.*)$"); + private static void flattenImport( - final DocumentBuilderFactory dfactory, Document doc, Node item) - throws MalformedURLException { + final DocumentBuilderFactory dfactory, Document doc, Node item) throws IOException { final String base = getBase(item); final String path = getPath(item); System.err.println("Import: " + base + ":" + path); if (base.equals("cldr")) { - if (path.startsWith("45/")) { - final String subpath = path.replaceFirst("45/", ""); - final File importDir = - new File( - CLDRConfig.getInstance().getCldrBaseDirectory(), - "keyboards/import"); - final File importFile = new File(importDir, subpath); - applyImportFile(dfactory, doc, item, path, importFile); + final Matcher m = CLDR_PREFIX_PATTERN.matcher(path); + if (m.matches()) { + final String subpath = m.group(2); + applyImportItem(dfactory, doc, item, path, subpath); } else { throw new IllegalArgumentException("Unknown cldr base: " + path); } @@ -105,13 +135,26 @@ private static void flattenImport( } } + private static void applyImportItem( + final DocumentBuilderFactory dfactory, + Document doc, + Node item, + final String path, + final String subpath) + throws IOException { + final File importDir = + new File(CLDRConfig.getInstance().getCldrBaseDirectory(), "keyboards/import"); + final File importFile = new File(importDir, subpath); + applyImportFile(dfactory, doc, item, path, importFile); + } + private static void applyImportFile( final DocumentBuilderFactory dfactory, Document doc, Node item, final String path, final File importFile) - throws MalformedURLException { + throws IOException { if (!importFile.exists()) { throw new IllegalArgumentException("File " + importFile + " does not exist"); } @@ -120,12 +163,10 @@ private static void applyImportFile( PathUtilities.getNormalizedPathString(importFile.getAbsolutePath()); // Force filerefs to be URI's if needed: note this is independent of any // other files - String docURI; - docURI = XMLValidator.filenameToURL(ifilename); Document importDoc = parseDocument( - new InputSource(docURI), ifilename, dfactory, getNullHandler(ifilename)); + getInputSource(ifilename), ifilename, dfactory, getNullHandler(ifilename)); System.err.println("Parsed import OK"); // Now perform the import @@ -164,32 +205,40 @@ private static void applyImportFile( // done } + private static final boolean DEBUG_XML_PARSE_ERRORS = false; + private static ErrorHandler getNullHandler(final String filename2) { ErrorHandler nullHandler = new ErrorHandler() { @Override public void warning(SAXParseException e) throws SAXException { - System.err.println(filename2 + ": Warning: " + e.getMessage()); + if (DEBUG_XML_PARSE_ERRORS) { + System.err.println(filename2 + ": Warning: " + e.getMessage()); + } } @Override public void error(SAXParseException e) throws SAXException { - int col = e.getColumnNumber(); - System.err.println( - filename2 - + ":" - + e.getLineNumber() - + (col >= 0 ? ":" + col : "") - + ": ERROR: Element " - + e.getPublicId() - + " is not valid because " - + e.getMessage()); + if (DEBUG_XML_PARSE_ERRORS) { + int col = e.getColumnNumber(); + System.err.println( + filename2 + + ":" + + e.getLineNumber() + + (col >= 0 ? ":" + col : "") + + ": ERROR: Element " + + e.getPublicId() + + " is not valid because " + + e.getMessage()); + } } @Override public void fatalError(SAXParseException e) throws SAXException { - System.err.println(filename2 + ": ERROR "); - throw e; + if (DEBUG_XML_PARSE_ERRORS) { + System.err.println(filename2 + ": ERROR "); + throw e; + } } }; return nullHandler; diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/util/CLDRPaths.java b/tools/cldr-code/src/main/java/org/unicode/cldr/util/CLDRPaths.java index a0b44bdaf7b..dba74946bfd 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/util/CLDRPaths.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/util/CLDRPaths.java @@ -168,6 +168,14 @@ public static final Path getDtd2MdDirectory() { public static final String LOG_DIRECTORY = CldrUtility.getPath(TMP_DIRECTORY, "logs/"); public static final String TEST_DIR = CldrUtility.getPath(CLDRPaths.TMP_DIRECTORY, "test/"); + /** root of the keyboards structure */ + public static final String KEYBOARDS_DIRECTORY = + CldrUtility.getPath(BASE_DIRECTORY, "keyboards"); + + /** the 3.0 directory, this contains the actual keyboards */ + public static final String KEYBOARDS_3_DIRECTORY = + CldrUtility.getPath(KEYBOARDS_DIRECTORY, "3.0"); + /** If the generated BAT files are to work, this needs to be set right */ public static final String COMPARE_PROGRAM = "\"C:\\Program Files (x86)\\Compare It!\\wincmp3.exe\""; diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/util/DoctypeXmlStreamWrapper.java b/tools/cldr-code/src/main/java/org/unicode/cldr/util/DoctypeXmlStreamWrapper.java index a4f5c5ee948..8d238786e6f 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/util/DoctypeXmlStreamWrapper.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/util/DoctypeXmlStreamWrapper.java @@ -1,11 +1,14 @@ package org.unicode.cldr.util; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.PushbackInputStream; import java.io.PushbackReader; import java.io.Reader; import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.regex.Matcher; @@ -42,9 +45,18 @@ public static InputSource wrap(InputSource src) throws IOException { src.setCharacterStream(wrap(r)); } else if (is != null) { src.setByteStream(wrap(is, src.getEncoding())); + } else if (src.getSystemId() != null && src.getSystemId().startsWith("file:")) { + try { + src.setByteStream( + wrap( + new FileInputStream(new URI(src.getSystemId()).getPath()), + src.getEncoding())); + } catch (URISyntaxException e) { + return src; // can't read, pass to caller + } } else { - throw new NullPointerException( - "Internal error: Character and Byte stream are both null"); + // return and let caller deal with it + return src; } return src; } diff --git a/docs/charts/keyboards/static/index.html b/tools/cldr-code/src/main/resources/org/unicode/cldr/util/data/keyboard-charts/index.html similarity index 100% rename from docs/charts/keyboards/static/index.html rename to tools/cldr-code/src/main/resources/org/unicode/cldr/util/data/keyboard-charts/index.html diff --git a/docs/charts/keyboards/static/keyboard-chart.css b/tools/cldr-code/src/main/resources/org/unicode/cldr/util/data/keyboard-charts/keyboard-chart.css similarity index 100% rename from docs/charts/keyboards/static/keyboard-chart.css rename to tools/cldr-code/src/main/resources/org/unicode/cldr/util/data/keyboard-charts/keyboard-chart.css diff --git a/docs/charts/keyboards/static/keyboard-chart.js b/tools/cldr-code/src/main/resources/org/unicode/cldr/util/data/keyboard-charts/keyboard-chart.js similarity index 82% rename from docs/charts/keyboards/static/keyboard-chart.js rename to tools/cldr-code/src/main/resources/org/unicode/cldr/util/data/keyboard-charts/keyboard-chart.js index 353ed9dfc4f..575f6f3f16e 100644 --- a/docs/charts/keyboards/static/keyboard-chart.js +++ b/tools/cldr-code/src/main/resources/org/unicode/cldr/util/data/keyboard-charts/keyboard-chart.js @@ -41,7 +41,8 @@ function mogrifyLayerList(layerList, keybag) { } function getImportFile(id) { - return _KeyboardData.imports[id["@_path"].split("/")[1]]; + return undefined; + // return _KeyboardData.imports[id["@_path"].split("/")[1]]; } function getImportKeys(id) { @@ -90,21 +91,21 @@ function getKeyboardKeys(id) { if (!keys) { throw Error(`No keys for ${id}`); } - let imports = [ - { - // add implied import - "@_base": "cldr", - "@_path": "45/keys-Latn-implied.xml", - }, - ...(_KeyboardData.keyboards[id].keyboard3.keys.import || []), - ]; + // let imports = [ + // { + // // add implied import + // "@_base": "cldr", + // "@_path": "45/keys-Latn-implied.xml", + // }, + // ...(_KeyboardData.keyboards[id].keyboard3.keys.import || []), + // ]; const importedKeys = []; - for (const fn of imports) { - for (const k of getImportKeys(fn)) { - importedKeys.push(k); - } - } + // for (const fn of imports) { + // for (const k of getImportKeys(fn)) { + // importedKeys.push(k); + // } + // } return mogrifyKeys([...importedKeys, ...keys]); } diff --git a/tools/pom.xml b/tools/pom.xml index f4ede1c6f9f..4484861e061 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -82,7 +82,6 @@ cldr-code cldr-apps cldr-rdf - ../docs/charts/keyboards