diff --git a/build.gradle.kts b/build.gradle.kts index 2ac8aed1b..1922b1277 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -5,13 +5,14 @@ // gradle clean build -PjavacRelease=21 // gradle clean build -PcustomVersion=1.0.0-10-asdf val customVersion = (project.findProperty("customVersion") ?: "") as String -val javacRelease = (project.findProperty("javacRelease") ?: "17") as String +val javacRelease = (project.findProperty("javacRelease") ?: "21") as String plugins { id("fr.brouillard.oss.gradle.jgitver") version "0.9.1" id("jacoco") id("java") id("maven-publish") + id("com.gradleup.shadow") version "9.0.0-rc1" } repositories { @@ -23,11 +24,11 @@ dependencies { implementation("com.lmax:disruptor:3.4.4") implementation("org.java-websocket:Java-WebSocket:1.5.3") implementation("org.jsoup:jsoup:1.16.1") - implementation("org.json:json:20211205") + implementation("org.json:json:20231013") implementation("com.j2html:j2html:1.6.0") implementation("commons-configuration:commons-configuration:1.10") implementation("commons-cli:commons-cli:1.5.0") - implementation("commons-io:commons-io:2.13.0") + implementation("commons-io:commons-io:2.14.0") implementation("org.apache.httpcomponents:httpclient:4.5.14") implementation("org.apache.httpcomponents:httpmime:4.5.14") implementation("org.apache.logging.log4j:log4j-api:2.20.0") @@ -36,6 +37,7 @@ dependencies { implementation("org.graalvm.js:js:22.3.2") testImplementation(enforcedPlatform("org.junit:junit-bom:5.10.0")) testImplementation("org.junit.jupiter:junit-jupiter") + testImplementation("org.mockito:mockito-core:5.+") testRuntimeOnly("org.junit.platform:junit-platform-launcher") } @@ -43,6 +45,11 @@ group = "com.rarchives.ripme" version = "1.7.94" description = "ripme" +java { + sourceCompatibility = JavaVersion.VERSION_21 + targetCompatibility = JavaVersion.VERSION_21 +} + jacoco { toolVersion = "0.8.12" } @@ -80,6 +87,10 @@ tasks.withType { }) } +tasks.shadowJar { + transform() +} + publishing { publications { create("maven") { @@ -105,6 +116,8 @@ tasks.test { includeEngines("junit-vintage") } finalizedBy(tasks.jacocoTestReport) // report is always generated after tests run + + jvmArgs("-javaagent:${classpath.find { it.name.contains("mockito") }?.absolutePath}") } tasks.register("testAll") { diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index df97d72b8..d4081da47 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index bd373a4e0..0d120c1c1 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -76,6 +76,9 @@ public static void main(String[] args) throws IOException { if (GraphicsEnvironment.isHeadless() || args.length > 0) { handleArguments(args); } else { + // Antialiasing hint, especially for Linux + System.setProperty("awt.useSystemAAFontSettings", "on"); + if (SystemUtils.IS_OS_MAC_OSX) { System.setProperty("apple.laf.useScreenMenuBar", "true"); System.setProperty("com.apple.mrj.application.apple.menu.about.name", "RipMe"); diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 0740f62c4..8b4ca631a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -13,11 +13,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -36,9 +32,6 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { private static final Logger logger = LogManager.getLogger(AbstractHTMLRipper.class); - private final Map itemsPending = Collections.synchronizedMap(new HashMap<>()); - private final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); - private final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); Document cachedFirstPage; protected AbstractHTMLRipper(URL url) throws IOException { @@ -76,10 +69,6 @@ protected List getDescriptionsFromPage(Document doc) throws IOException protected abstract void downloadURL(URL url, int index); - protected DownloadThreadPool getThreadPool() { - return null; - } - protected boolean keepSortOrder() { return true; } @@ -121,7 +110,7 @@ protected boolean pageContainsAlbums(URL url) { @Override public void rip() throws IOException, URISyntaxException { - int index = 0; + int imageIndex = 0; int textindex = 0; logger.info("Retrieving " + this.url); sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm()); @@ -176,9 +165,10 @@ public void rip() throws IOException, URISyntaxException { } for (String imageURL : imageURLs) { - index += 1; - logger.debug("Found image url #" + index + ": '" + imageURL + "'"); - downloadURL(new URI(imageURL).toURL(), index); + imageIndex += 1; + logger.debug("Found image url #{} of album {}: {}", imageIndex, this.url, imageURL); + setItemsTotal(Math.max(getItemsTotal(), imageIndex)); + downloadURL(new URI(imageURL).toURL(), imageIndex); if (isStopped() || isThisATest()) { break; } @@ -206,7 +196,7 @@ public void rip() throws IOException, URISyntaxException { workingDir.getCanonicalPath() + "" + File.separator - + getPrefix(index) + + getPrefix(imageIndex) + (tempDesc.length > 1 ? tempDesc[1] : filename) + ".txt").exists(); @@ -236,12 +226,17 @@ public void rip() throws IOException, URISyntaxException { } } - // If they're using a thread pool, wait for it. - if (getThreadPool() != null) { - logger.debug("Waiting for threadpool " + getThreadPool().getClass().getName()); - getThreadPool().waitForThreads(); + logger.info("All items queued; total items: {}; url: {}", imageIndex, url); + + // Final total item count is now known + setItemsTotal(imageIndex); + + if (getCrawlerThreadPool() != null) { + logger.debug("Waiting for crawler threadpool: {}", url); + getCrawlerThreadPool().waitForThreads(imageIndex, shouldStop, url); } - waitForThreads(); + + waitForRipperThreads(); } /** @@ -338,68 +333,6 @@ protected boolean allowDuplicates() { return false; } - @Override - /* - Returns total amount of files attempted. - */ - public int getCount() { - return itemsCompleted.size() + itemsErrored.size(); - } - - @Override - /* - Queues multiple URLs of single images to download from a single Album URL - */ - public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { - // Only download one file if this is a test. - if (isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { - stop(); - itemsPending.clear(); - return false; - } - if (!allowDuplicates() - && ( itemsPending.containsKey(url) - || itemsCompleted.containsKey(url) - || itemsErrored.containsKey(url) )) { - // Item is already downloaded/downloading, skip it. - logger.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs)); - return false; - } - if (shouldIgnoreURL(url)) { - sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); - return false; - } - if (Utils.getConfigBoolean("urls_only.save", false)) { - // Output URL to file - Path urlFile = Paths.get(this.workingDir + "/urls.txt"); - String text = url.toExternalForm() + System.lineSeparator(); - try { - Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND); - itemsCompleted.put(url, urlFile); - } catch (IOException e) { - logger.error("Error while writing to " + urlFile, e); - } - } - else { - itemsPending.put(url, saveAs.toFile()); - DownloadFileThread dft = new DownloadFileThread(url, saveAs.toFile(), this, getFileExtFromMIME); - if (referrer != null) { - dft.setReferrer(referrer); - } - if (cookies != null) { - dft.setCookies(cookies); - } - threadPool.addThread(dft); - } - - return true; - } - - @Override - public boolean addURLToDownload(URL url, Path saveAs) { - return addURLToDownload(url, saveAs, null, null, false); - } - /** * Queues image to be downloaded and saved. * Uses filename from URL to decide filename. @@ -413,72 +346,6 @@ protected boolean addURLToDownload(URL url) { return addURLToDownload(url, "", ""); } - @Override - /* - Cleans up & tells user about successful download - */ - public void downloadCompleted(URL url, Path saveAs) { - if (observer == null) { - return; - } - try { - String path = Utils.removeCWD(saveAs); - RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); - itemsPending.remove(url); - itemsCompleted.put(url, saveAs); - observer.update(this, msg); - - checkIfComplete(); - } catch (Exception e) { - logger.error("Exception while updating observer: ", e); - } - } - - @Override - /* - * Cleans up & tells user about failed download. - */ - public void downloadErrored(URL url, String reason) { - if (observer == null) { - return; - } - itemsPending.remove(url); - itemsErrored.put(url, reason); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason)); - - checkIfComplete(); - } - - @Override - /* - Tells user that a single file in the album they wish to download has - already been downloaded in the past. - */ - public void downloadExists(URL url, Path file) { - if (observer == null) { - return; - } - - itemsPending.remove(url); - itemsCompleted.put(url, file); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " already saved as " + file)); - - checkIfComplete(); - } - - /** - * Notifies observers and updates state if all files have been ripped. - */ - @Override - protected void checkIfComplete() { - if (observer == null) { - return; - } - if (itemsPending.isEmpty()) { - super.checkIfComplete(); - } - } - /** * Sets directory to save all ripped files to. * @param url @@ -515,22 +382,20 @@ public void setWorkingDir(URL url) throws IOException, URISyntaxException { */ @Override public int getCompletionPercentage() { - double total = itemsPending.size() + itemsErrored.size() + itemsCompleted.size(); - return (int) (100 * ( (total - itemsPending.size()) / total)); + double total = getTotalCount(); + if (total == 0) { + return 0; + } + return (int) (100 * ( (itemsCompleted.size() + itemsErrored.size()) / total)); } - /** - * @return - * Human-readable information on the status of the current rip. - */ @Override - public String getStatusText() { - return getCompletionPercentage() + - "% " + - "- Pending: " + itemsPending.size() + - ", Completed: " + itemsCompleted.size() + - ", Errored: " + itemsErrored.size(); + public int getPendingCount() { + DownloadThreadPool threadPool = getRipperThreadPool(); + if (threadPool != null) { + return threadPool.getPendingThreadCount(); + } + return itemsPending.size(); } - } diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index a49084c63..d2d6931b5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -11,10 +11,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -31,10 +28,6 @@ public abstract class AbstractJSONRipper extends AbstractRipper { private static final Logger logger = LogManager.getLogger(AbstractJSONRipper.class); - private Map itemsPending = Collections.synchronizedMap(new HashMap()); - private Map itemsCompleted = Collections.synchronizedMap(new HashMap()); - private Map itemsErrored = Collections.synchronizedMap(new HashMap()); - protected AbstractJSONRipper(URL url) throws IOException { super(url); } @@ -49,9 +42,6 @@ protected JSONObject getNextPage(JSONObject doc) throws IOException, URISyntaxEx } protected abstract List getURLsFromJSON(JSONObject json); protected abstract void downloadURL(URL url, int index); - private DownloadThreadPool getThreadPool() { - return null; - } protected boolean keepSortOrder() { return true; @@ -69,7 +59,7 @@ public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException @Override public void rip() throws IOException, URISyntaxException { - int index = 0; + int imageIndex = 0; logger.info("Retrieving " + this.url); sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm()); JSONObject json = getFirstPage(); @@ -98,9 +88,10 @@ public void rip() throws IOException, URISyntaxException { break; } - index += 1; - logger.debug("Found image url #" + index+ ": " + imageURL); - downloadURL(new URI(imageURL).toURL(), index); + imageIndex += 1; + logger.debug("Found image url #{} of album {}: {}", imageIndex, this.url, imageURL); + setItemsTotal(Math.max(getItemsTotal(), imageIndex)); + downloadURL(new URI(imageURL).toURL(), imageIndex); } if (isStopped() || isThisATest()) { @@ -116,12 +107,17 @@ public void rip() throws IOException, URISyntaxException { } } - // If they're using a thread pool, wait for it. - if (getThreadPool() != null) { - logger.debug("Waiting for threadpool " + getThreadPool().getClass().getName()); - getThreadPool().waitForThreads(); + logger.info("All items queued; total items: {}; url: {}", imageIndex, url); + + // Final total item count is now known + setItemsTotal(imageIndex); + + if (getCrawlerThreadPool() != null) { + logger.debug("Waiting for crawler threadpool: {}", url); + getCrawlerThreadPool().waitForThreads(imageIndex, shouldStop, url); } - waitForThreads(); + + waitForRipperThreads(); } protected String getPrefix(int index) { @@ -140,68 +136,6 @@ protected boolean allowDuplicates() { return false; } - @Override - /** - * Returns total amount of files attempted. - */ - public int getCount() { - return itemsCompleted.size() + itemsErrored.size(); - } - - @Override - /** - * Queues multiple URLs of single images to download from a single Album URL - */ - public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { - // Only download one file if this is a test. - if (super.isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { - stop(); - itemsPending.clear(); - return false; - } - if (!allowDuplicates() - && ( itemsPending.containsKey(url) - || itemsCompleted.containsKey(url) - || itemsErrored.containsKey(url) )) { - // Item is already downloaded/downloading, skip it. - logger.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs)); - return false; - } - if (shouldIgnoreURL(url)) { - sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); - return false; - } - if (Utils.getConfigBoolean("urls_only.save", false)) { - // Output URL to file - Path urlFile = Paths.get(this.workingDir + "/urls.txt"); - String text = url.toExternalForm() + System.lineSeparator(); - try { - Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND); - itemsCompleted.put(url, urlFile); - } catch (IOException e) { - logger.error("Error while writing to " + urlFile, e); - } - } - else { - itemsPending.put(url, saveAs.toFile()); - DownloadFileThread dft = new DownloadFileThread(url, saveAs.toFile(), this, getFileExtFromMIME); - if (referrer != null) { - dft.setReferrer(referrer); - } - if (cookies != null) { - dft.setCookies(cookies); - } - threadPool.addThread(dft); - } - - return true; - } - - @Override - public boolean addURLToDownload(URL url, Path saveAs) { - return addURLToDownload(url, saveAs, null, null, false); - } - /** * Queues image to be downloaded and saved. * Uses filename from URL to decide filename. @@ -215,72 +149,6 @@ protected boolean addURLToDownload(URL url) { return addURLToDownload(url, "", ""); } - @Override - /** - * Cleans up & tells user about successful download - */ - public void downloadCompleted(URL url, Path saveAs) { - if (observer == null) { - return; - } - try { - String path = Utils.removeCWD(saveAs); - RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); - itemsPending.remove(url); - itemsCompleted.put(url, saveAs); - observer.update(this, msg); - - checkIfComplete(); - } catch (Exception e) { - logger.error("Exception while updating observer: ", e); - } - } - - @Override - /** - * Cleans up & tells user about failed download. - */ - public void downloadErrored(URL url, String reason) { - if (observer == null) { - return; - } - itemsPending.remove(url); - itemsErrored.put(url, reason); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason)); - - checkIfComplete(); - } - - @Override - /** - * Tells user that a single file in the album they wish to download has - * already been downloaded in the past. - */ - public void downloadExists(URL url, Path file) { - if (observer == null) { - return; - } - - itemsPending.remove(url); - itemsCompleted.put(url, file); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " already saved as " + file)); - - checkIfComplete(); - } - - /** - * Notifies observers and updates state if all files have been ripped. - */ - @Override - protected void checkIfComplete() { - if (observer == null) { - return; - } - if (itemsPending.isEmpty()) { - super.checkIfComplete(); - } - } - /** * Sets directory to save all ripped files to. * @param url @@ -315,24 +183,20 @@ public void setWorkingDir(URL url) throws IOException, URISyntaxException { */ @Override public int getCompletionPercentage() { - double total = itemsPending.size() + itemsErrored.size() + itemsCompleted.size(); - return (int) (100 * ( (total - itemsPending.size()) / total)); + double total = getTotalCount(); + if (total == 0) { + return 0; + } + return (int) (100 * ( (itemsCompleted.size() + itemsErrored.size()) / total)); } - /** - * @return - * Human-readable information on the status of the current rip. - */ @Override - public String getStatusText() { - StringBuilder sb = new StringBuilder(); - sb.append(getCompletionPercentage()) - .append("% ") - .append("- Pending: " ).append(itemsPending.size()) - .append(", Completed: ").append(itemsCompleted.size()) - .append(", Errored: " ).append(itemsErrored.size()); - return sb.toString(); + public int getPendingCount() { + DownloadThreadPool threadPool = getRipperThreadPool(); + if (threadPool != null) { + return threadPool.getPendingThreadCount(); + } + return itemsPending.size(); } - } diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index f1ed45600..06fe8cab2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -1,6 +1,5 @@ package com.rarchives.ripme.ripper; -import java.awt.Desktop; import java.io.BufferedWriter; import java.io.File; import java.io.FileNotFoundException; @@ -11,16 +10,14 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Observable; -import java.util.Random; -import java.util.Scanner; +import java.nio.file.StandardOpenOption; +import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -45,6 +42,34 @@ public abstract class AbstractRipper implements RipperInterface, Runnable { private static final Logger logger = LogManager.getLogger(AbstractRipper.class); + + // For albums + protected final Set itemsPending = Collections.synchronizedSet(new HashSet<>()); + protected final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); + protected final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); + protected final Map itemsSkipped = Collections.synchronizedMap(new HashMap<>()); + + /** + * Rippers should set itemsTotal to the best known number of total items, + * if known at the start of the rip, e.g. in getFirstPage(). + * The best known number might be indicated on the album page, + * or calculated by the number of pages and the number of items per page. + * Once the last item is seen by the HTML or JSON crawler, the final value is set. + */ + private final AtomicInteger itemsTotal = new AtomicInteger(0); + + /** + * If an album has a duplicate RipUrlId (e.g. the same image linked twice), + * duplicates can't be counted by itemsPending, but {@link #waitForRipperThreads()} needs + * to know that the ripper has seen each link crawled. + */ + private final AtomicInteger itemsSeen = new AtomicInteger(0); + + /** For individual files. See {@link #useByteProgessBar()} */ + protected long bytesCompleted = 0; + /** For individual files. See {@link #useByteProgessBar()} */ + protected long bytesTotal = 1; // avoid divide by 0 + private final String URLHistoryFile = Utils.getURLHistoryFile(); public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36"; @@ -53,10 +78,11 @@ public abstract class AbstractRipper protected URL url; protected File workingDir; - DownloadThreadPool threadPool; + private DownloadThreadPool ripperThreadPool; + private DownloadThreadPool crawlerThreadPool; RipStatusHandler observer = null; - private boolean completed = true; + private final AtomicBoolean completed = new AtomicBoolean(false); public abstract void rip() throws IOException, URISyntaxException; @@ -64,13 +90,16 @@ public abstract class AbstractRipper public abstract String getGID(URL url) throws MalformedURLException, URISyntaxException; + protected abstract boolean allowDuplicates(); + public boolean hasASAPRipping() { return false; } // Everytime addUrlToDownload skips a already downloaded url this increases by 1 public int alreadyDownloadedUrls = 0; - private final AtomicBoolean shouldStop = new AtomicBoolean(false); + protected final AtomicBoolean shouldStop = new AtomicBoolean(false); + protected final AtomicBoolean shouldPanic = new AtomicBoolean(false); private static boolean thisIsATest = false; public void stop() { @@ -78,6 +107,16 @@ public void stop() { shouldStop.set(true); } + public void panic() { + logger.trace("panic()"); + shouldStop.set(true); + shouldPanic.set(true); + } + + public boolean isPanicked() { + return shouldPanic.get(); + } + public boolean isStopped() { return shouldStop.get(); } @@ -88,6 +127,23 @@ protected void stopCheck() throws IOException { } } + /** + * Used for file downloads. Used by {@link #addURLToDownload(TokenedUrlGetter, RipUrlId, Path, String, String, Map, Boolean)} + */ + protected DownloadThreadPool getRipperThreadPool() { + return ripperThreadPool; + } + + /** + * Used by Rippers to crawl file pages.
+ * After the last file page is crawled and all threads are queued to the ripper thread pool, + * {@link #rip()} terminates the crawler thread pool.
+ * After the crawler thread pool is finished, {@link #rip()} terminates the ripper thread pool. + */ + protected DownloadThreadPool getCrawlerThreadPool() { + return crawlerThreadPool; + } + /** * Adds a URL to the url history file * @@ -215,7 +271,8 @@ public void setup() throws IOException, URISyntaxException { // ctx.reconfigure(); // ctx.updateLoggers(); - this.threadPool = new DownloadThreadPool(); + this.ripperThreadPool = new DownloadThreadPool(getClass().getSimpleName() + "-ripper-" + getGID(url)); + this.crawlerThreadPool = new DownloadThreadPool(getClass().getSimpleName() + "-crawler"); } public void setObserver(RipStatusHandler obs) { @@ -229,7 +286,9 @@ public void setObserver(RipStatusHandler obs) { * @param saveAs Path of the local file to save the content to. * @return True on success, false on failure. */ - public abstract boolean addURLToDownload(URL url, Path saveAs); + public boolean addURLToDownload(URL url, Path saveAs) { + return addURLToDownload(url, saveAs, null, null, false); + } /** * Queues image to be downloaded and saved. @@ -242,8 +301,86 @@ public void setObserver(RipStatusHandler obs) { * @return True if downloaded successfully * False if failed to download */ - protected abstract boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, - Boolean getFileExtFromMIME); + public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { + TokenedUrlGetter tug = () -> url; + RipUrlId ripUrlId = new RipUrlId(getClass(), getHost(), url); + Path directory = saveAs.getParent(); + String filename = saveAs.getFileName().toString(); + return addURLToDownload(tug, ripUrlId, directory, filename, referrer, cookies, getFileExtFromMIME); + } + + protected boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId) { + Path workingDir = getWorkingDir().toPath(); + String filename = null; + return addURLToDownload(tug, ripUrlId, workingDir, filename, null, null, false); + } + + protected boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String referrer, Map cookies, Boolean getFileExtFromMIME) { + return addURLToDownload(tug, ripUrlId, directory, null, referrer, cookies, getFileExtFromMIME); + } + + /** + * Queues multiple URLs of single images to download from a single Album URL + */ + public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, String referrer, Map cookies, Boolean getFileExtFromMIME) { + // Only download one file if this is a test. + if (isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { + stop(); + itemsPending.clear(); + return false; + } + itemsSeen.incrementAndGet(); + + if (!allowDuplicates() + && ( itemsPending.contains(ripUrlId) + || itemsCompleted.containsKey(ripUrlId) + || itemsErrored.containsKey(ripUrlId) )) { + // Item is already downloaded/downloading, skip it. + // TODO print path if in itemsCompleted or itemsErrored + logger.info("[!] Skipping " + ripUrlId + " -- already attempted: " + Utils.removeCWD(directory)); + return false; + } + + if (Utils.getConfigBoolean("urls_only.save", false)) { + // Output URL to file + Path urlFile = Paths.get(this.workingDir + "/urls.txt"); + URL url = null; + try { + url = tug.getTokenedUrl(); + } catch (IOException | URISyntaxException e) { + logger.error("Unable to get URL for {}", ripUrlId, e); + itemsErrored.put(ripUrlId, e.getMessage()); + return false; + } + if (AbstractRipper.shouldIgnoreExtension(url)) { + sendUpdate(STATUS.DOWNLOAD_SKIP, Utils.getLocalizedString("skipping.ignored.extension") + ": " + url.toExternalForm()); + return false; + } + String text = url.toExternalForm() + System.lineSeparator(); + try { + Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND); + itemsCompleted.put(ripUrlId, urlFile); + } catch (IOException e) { + logger.error("Error while writing to " + urlFile, e); + return false; + } + return true; + } + + itemsPending.add(ripUrlId); + DownloadFileThread dft = new DownloadFileThread(tug, ripUrlId, directory, filename, this, getFileExtFromMIME); + if (referrer != null) { + dft.setReferrer(referrer); + } + if (cookies != null) { + dft.setCookies(cookies); + } + getRipperThreadPool().addThread(dft); + + return true; + } + + /** * Queues image to be downloaded and saved. @@ -478,11 +615,24 @@ public static String getFileName(URL url, String prefix, String fileName, String /** * Waits for downloading threads to complete. */ - protected void waitForThreads() { - logger.debug("Waiting for threads to finish"); - completed = false; - threadPool.waitForThreads(); - checkIfComplete(); + protected void waitForRipperThreads() { + waitForRipperThreads(true); + } + + protected void waitForRipperThreads(boolean notifyComplete) { + logger.debug("Waiting for threads to finish; url: {}", url); + if (!notifyComplete) { + setItemsTotal(0); + } + ripperThreadPool.waitForThreads(() -> { + boolean finished = shouldStop.get() || (itemsSeen.get() >= itemsTotal.get() && itemsPending.isEmpty()); + logger.trace("ripperThreadPool: are threads finished? {} url: {} shouldStop: {}; itemsPending.size(): {}; itemsCompleted.size(): {}; itemsErrored.size(): {}; itemsSkipped.size(): {}; itemsTotal: {}; itemsSeen: {}", + finished, url, shouldStop, itemsPending.size(), itemsCompleted.size(), itemsErrored.size(), itemsSkipped.size(), itemsTotal, itemsSeen); + return finished; + }, url); + if (notifyComplete) { + notifyComplete(); + } } /** @@ -500,44 +650,93 @@ public void retrievingSource(String url) { /** * Notifies observers that a file download has completed. * - * @param url URL that was completed. - * @param saveAs Where the downloaded file is stored. + * @param ripUrlId The RipUrlId that was completed. + * @param saveAs Where the downloaded file is stored. */ - public abstract void downloadCompleted(URL url, Path saveAs); + protected void downloadCompleted(RipUrlId ripUrlId, Path saveAs) { + if (observer == null) { + return; + } + try { + String path = Utils.removeCWD(saveAs); + RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); + itemsPending.remove(ripUrlId); + itemsCompleted.put(ripUrlId, saveAs); + observer.update(this, msg); + + //checkIfComplete(); + } catch (Exception e) { + logger.error("Exception while updating observer: ", e); + } + } + + /** + * Notifies observers that a file could not be downloaded (includes a reason). + */ + protected void downloadErrored(RipUrlId ripUrlId, String reason) { + if (observer == null) { + return; + } + itemsPending.remove(ripUrlId); + itemsErrored.put(ripUrlId, reason); + observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, ripUrlId + " : " + reason)); + + //checkIfComplete(); + } /** * Notifies observers that a file could not be downloaded (includes a reason). */ - public abstract void downloadErrored(URL url, String reason); + protected void downloadSkipped(RipUrlId ripUrlId, String reason) { + if (observer == null) { + return; + } + itemsPending.remove(ripUrlId); + //itemsSkipped.put(ripUrlId, reason); + itemsCompleted.put(ripUrlId, null); // TODO use itemsSkipped and make the progress bar display it as completed + observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_SKIP, ripUrlId + " : " + reason)); + + //checkIfComplete(); + } /** * Notify observers that a download could not be completed, * but was not technically an "error". */ - public abstract void downloadExists(URL url, Path file); + protected void downloadExists(RipUrlId ripUrlId, Path file) { + if (observer == null) { + return; + } + + itemsPending.remove(ripUrlId); + itemsCompleted.put(ripUrlId, file); + observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, Utils.getLocalizedString("0.already.saved.as.1", ripUrlId, file))); + + //checkIfComplete(); + } /** * @return Number of files downloaded. */ - int getCount() { - return 1; + public int getCount() { + return itemsCompleted.size() + itemsErrored.size(); } /** * Notifies observers and updates state if all files have been ripped. */ - void checkIfComplete() { + protected void notifyComplete() { if (observer == null) { logger.debug("observer is null"); return; } - if (!completed) { - completed = true; - logger.info(" Rip completed!"); + if (!completed.getAndSet(true)) { + logger.info(" Rip of {} completed!", getURL()); RipStatusComplete rsc = new RipStatusComplete(workingDir.toPath(), getCount()); RipStatusMessage msg = new RipStatusMessage(STATUS.RIP_COMPLETE, rsc); + logger.debug("Sending RIP_COMPLETE: url: {}", getURL()); observer.update(this, msg); // we do not care if the rollingfileappender is active, @@ -551,7 +750,7 @@ void checkIfComplete() { if (Utils.getConfigBoolean("urls_only.save", false)) { String urlFile = this.workingDir + File.separator + "urls.txt"; try { - Desktop.getDesktop().open(new File(urlFile)); + Utils.open(new File(urlFile)); } catch (IOException e) { logger.warn("Error while opening " + urlFile, e); } @@ -658,26 +857,23 @@ public void sendUpdate(STATUS status, Object message) { */ public abstract int getCompletionPercentage(); - /** - * @return Text for status - */ - public abstract String getStatusText(); - /** * Rips the album when the thread is invoked. */ public void run() { try { + logger.info("Rip started: {}", getURL()); rip(); } catch (HttpStatusException e) { logger.error("Got exception while running ripper:", e); - waitForThreads(); + waitForRipperThreads(false); sendUpdate(STATUS.RIP_ERRORED, "HTTP status code " + e.getStatusCode() + " for URL " + e.getUrl()); } catch (Exception e) { logger.error("Got exception while running ripper:", e); - waitForThreads(); + waitForRipperThreads(false); sendUpdate(STATUS.RIP_ERRORED, e.getMessage()); } finally { + logger.info("Rip ended: {}", getURL()); cleanup(); } } @@ -761,12 +957,12 @@ protected boolean sleep(int milliseconds) { } } - public void setBytesTotal(int bytes) { - // Do nothing + protected void setBytesTotal(long bytes) { + this.bytesTotal = bytes; } - public void setBytesCompleted(int bytes) { - // Do nothing + protected void setBytesCompleted(long bytes) { + this.bytesCompleted = bytes; } /** Methods for detecting when we're running a test. */ @@ -780,7 +976,7 @@ protected static boolean isThisATest() { } // If true ripme uses a byte progress bar - protected boolean useByteProgessBar() { + public boolean useByteProgessBar() { return false; } @@ -789,7 +985,7 @@ protected boolean tryResumeDownload() { return false; } - protected boolean shouldIgnoreURL(URL url) { + protected static boolean shouldIgnoreExtension(URL url) { final String[] ignoredExtensions = Utils.getConfigStringArray("download.ignore_extensions"); if (ignoredExtensions == null || ignoredExtensions.length == 0) return false; // nothing ignored @@ -804,4 +1000,65 @@ protected boolean shouldIgnoreURL(URL url) { } return false; } + + public void setThreadPoolSize(int size) { + ripperThreadPool.setThreadPoolSize(size); + } + + public int getPendingCount() { + DownloadThreadPool threadPool = getRipperThreadPool(); + if (threadPool != null) { + return threadPool.getPendingThreadCount(); + } + return itemsPending.size(); + } + + public int getCompletedCount() { + return itemsCompleted.size(); + } + + public int getErroredCount() { + return itemsErrored.size(); + } + + public int getActiveCount() { + return ripperThreadPool.getActiveThreadCount(); + } + + public long getBytesTotal() { + return bytesTotal; + } + + public long getBytesCompleted() { + return bytesCompleted; + } + + /** + * Gets the best estimate of total items. + */ + public int getTotalCount() { + if (itemsTotal.get() == 0) { + return itemsPending.size() + itemsErrored.size() + itemsCompleted.size(); + } + return itemsTotal.get(); + } + + /** + * Gets the asserted number of total items, or 0 if unknown. + * Possibly useful in rippers. + * MainWindow probably wants {@link #getTotalCount()} + */ + protected int getItemsTotal() { + return itemsTotal.get(); + } + + /** + * For use in rippers to update the best estimate of total items. + */ + protected void setItemsTotal(int itemsTotal) { + if (itemsTotal < 0) { + throw new IllegalArgumentException("itemsTotal cannot be negative. Use 0 for unknown."); + } + this.itemsTotal.set(itemsTotal); + } } diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractSingleFileRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractSingleFileRipper.java index f1f8be41b..273d86367 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractSingleFileRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractSingleFileRipper.java @@ -11,33 +11,16 @@ * to help cut down on copy pasted code */ public abstract class AbstractSingleFileRipper extends AbstractHTMLRipper { - private int bytesTotal = 1; - private int bytesCompleted = 1; protected AbstractSingleFileRipper(URL url) throws IOException { super(url); } - @Override - public String getStatusText() { - return Utils.getByteStatusText(getCompletionPercentage(), bytesCompleted, bytesTotal); - } - @Override public int getCompletionPercentage() { return (int) (100 * (bytesCompleted / (float) bytesTotal)); } - @Override - public void setBytesTotal(int bytes) { - this.bytesTotal = bytes; - } - - @Override - public void setBytesCompleted(int bytes) { - this.bytesCompleted = bytes; - } - @Override public boolean useByteProgessBar() {return true;} } diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index bda3bf6fb..552f78709 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -10,9 +10,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; +import java.util.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -32,10 +30,6 @@ public abstract class AlbumRipper extends AbstractRipper { private static final Logger logger = LogManager.getLogger(AlbumRipper.class); - private Map itemsPending = Collections.synchronizedMap(new HashMap()); - private Map itemsCompleted = Collections.synchronizedMap(new HashMap()); - private Map itemsErrored = Collections.synchronizedMap(new HashMap()); - protected AlbumRipper(URL url) throws IOException { super(url); } @@ -50,147 +44,6 @@ protected boolean allowDuplicates() { return false; } - @Override - /** - * Returns total amount of files attempted. - */ - public int getCount() { - return itemsCompleted.size() + itemsErrored.size(); - } - - @Override - /** - * Queues multiple URLs of single images to download from a single Album URL - */ - public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { - // Only download one file if this is a test. - if (super.isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { - stop(); - itemsPending.clear(); - return false; - } - if (!allowDuplicates() - && ( itemsPending.containsKey(url) - || itemsCompleted.containsKey(url) - || itemsErrored.containsKey(url) )) { - // Item is already downloaded/downloading, skip it. - logger.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs)); - return false; - } - if (shouldIgnoreURL(url)) { - sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); - return false; - } - if (Utils.getConfigBoolean("urls_only.save", false)) { - // Output URL to file - Path urlFile = Paths.get(this.workingDir + "/urls.txt"); - String text = url.toExternalForm() + System.lineSeparator(); - try { - Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND); - itemsCompleted.put(url, urlFile); - } catch (IOException e) { - logger.error("Error while writing to " + urlFile, e); - } - } - else { - itemsPending.put(url, saveAs.toFile()); - DownloadFileThread dft = new DownloadFileThread(url, saveAs.toFile(), this, getFileExtFromMIME); - if (referrer != null) { - dft.setReferrer(referrer); - } - if (cookies != null) { - dft.setCookies(cookies); - } - threadPool.addThread(dft); - } - - return true; - } - - @Override - public boolean addURLToDownload(URL url, Path saveAs) { - return addURLToDownload(url, saveAs, null, null, false); - } - - /** - * Queues image to be downloaded and saved. - * Uses filename from URL to decide filename. - * @param url - * URL to download - * @return - * True on success - */ - protected boolean addURLToDownload(URL url) { - // Use empty prefix and empty subdirectory - return addURLToDownload(url, "", ""); - } - - @Override - /** - * Cleans up & tells user about successful download - */ - public void downloadCompleted(URL url, Path saveAs) { - if (observer == null) { - return; - } - try { - String path = Utils.removeCWD(saveAs); - RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); - itemsPending.remove(url); - itemsCompleted.put(url, saveAs); - observer.update(this, msg); - - checkIfComplete(); - } catch (Exception e) { - logger.error("Exception while updating observer: ", e); - } - } - - @Override - /** - * Cleans up & tells user about failed download. - */ - public void downloadErrored(URL url, String reason) { - if (observer == null) { - return; - } - itemsPending.remove(url); - itemsErrored.put(url, reason); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason)); - - checkIfComplete(); - } - - @Override - /** - * Tells user that a single file in the album they wish to download has - * already been downloaded in the past. - */ - public void downloadExists(URL url, Path file) { - if (observer == null) { - return; - } - - itemsPending.remove(url); - itemsCompleted.put(url, file); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " already saved as " + file)); - - checkIfComplete(); - } - - /** - * Notifies observers and updates state if all files have been ripped. - */ - @Override - protected void checkIfComplete() { - if (observer == null) { - return; - } - if (itemsPending.isEmpty()) { - super.checkIfComplete(); - } - } - /** * Sets directory to save all ripped files to. * @param url @@ -232,22 +85,11 @@ public void setWorkingDir(URL url) throws IOException, URISyntaxException { */ @Override public int getCompletionPercentage() { - double total = itemsPending.size() + itemsErrored.size() + itemsCompleted.size(); - return (int) (100 * ( (total - itemsPending.size()) / total)); + double total = getTotalCount(); + if (total == 0) { + return 0; + } + return (int) (100 * ( (itemsCompleted.size() + itemsErrored.size()) / total)); } - /** - * @return - * Human-readable information on the status of the current rip. - */ - @Override - public String getStatusText() { - StringBuilder sb = new StringBuilder(); - sb.append(getCompletionPercentage()) - .append("% ") - .append("- Pending: " ).append(itemsPending.size()) - .append(", Completed: ").append(itemsCompleted.size()) - .append(", Errored: " ).append(itemsErrored.size()); - return sb.toString(); - } } diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index e9c6f2427..41c12e7cf 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -2,7 +2,9 @@ import java.io.*; import java.net.*; +import java.nio.file.FileStore; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; import java.util.HashMap; @@ -23,13 +25,14 @@ */ class DownloadFileThread implements Runnable { private static final Logger logger = LogManager.getLogger(DownloadFileThread.class); + private final TokenedUrlGetter tokenedUrlGetter; // Some URLs may be valid for a limited time. This should get a fresh url + private final RipUrlId ripUrlId; private String referrer = ""; private Map cookies = new HashMap<>(); - private final URL url; - private File saveAs; - private final String prettySaveAs; + private final Path directory; + private String filename; private final AbstractRipper observer; private final int retries; private final Boolean getFileExtFromMIME; @@ -37,11 +40,13 @@ class DownloadFileThread implements Runnable { private final int TIMEOUT; private final int retrySleep; - public DownloadFileThread(URL url, File saveAs, AbstractRipper observer, Boolean getFileExtFromMIME) { + + public DownloadFileThread(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, AbstractRipper observer, Boolean getFileExtFromMIME) { super(); - this.url = url; - this.saveAs = saveAs; - this.prettySaveAs = Utils.removeCWD(saveAs.toPath()); + this.tokenedUrlGetter = tug; + this.ripUrlId = ripUrlId; + this.directory = directory; + this.filename = filename; this.observer = observer; this.retries = Utils.getConfigInteger("download.retries", 1); this.TIMEOUT = Utils.getConfigInteger("download.timeout", 60000); @@ -63,49 +68,86 @@ public void setCookies(Map cookies) { */ @Override public void run() { + + if (observer.isStopped()) { + // TODO add handler for graceful stop + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("download.interrupted")); + return; + } + + URL url = null; + try { + url = tokenedUrlGetter.getTokenedUrl(); + } catch (HttpStatusException e) { + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("failed.to.get.url.for.0", ripUrlId)); + logger.error("[!] Failed to get URL for " + ripUrlId); + return; // do not retry + } catch (IOException | URISyntaxException e) { + logger.error("[!] Failed to get URL for " + ripUrlId, e); + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("failed.to.get.url.for.0", ripUrlId)); + return; // do not retry + } + if (filename == null) { + // Strip token query parameters + filename = Path.of(url.getPath()).getFileName().toString(); + } // First thing we make sure the file name doesn't have any illegal chars in it - saveAs = new File( - saveAs.getParentFile().getAbsolutePath() + File.separator + Utils.sanitizeSaveAs(saveAs.getName())); + filename = Utils.sanitizeSaveAs(filename); + if (AbstractRipper.shouldIgnoreExtension(url)) { + observer.sendUpdate(STATUS.DOWNLOAD_SKIP, Utils.getLocalizedString("skipping.ignored.extension") + ": " + url.toExternalForm()); + return; + } + + if (!Files.exists(directory)) { + logger.info("[+] Creating directory: " + directory); + try { + Files.createDirectories(directory); + } catch (IOException e) { + logger.error("Error creating directory", e); + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("error.creating.directory") + ": " + directory + " ; " + e.getMessage()); + return; + } + } + + File saveAs = directory.resolve(filename).toFile(); + String prettySaveAs = Utils.removeCWD(saveAs.toPath()); + long fileSize = 0; - int bytesTotal; - int bytesDownloaded = 0; + long bytesTotal = 0; + long bytesDownloaded = 0; if (saveAs.exists() && observer.tryResumeDownload()) { fileSize = saveAs.length(); } - try { - observer.stopCheck(); - } catch (IOException e) { - observer.downloadErrored(url, Utils.getLocalizedString("download.interrupted")); - return; - } + if (saveAs.exists() && !observer.tryResumeDownload() && !getFileExtFromMIME || Utils.fuzzyExists(Paths.get(saveAs.getParent()), saveAs.getName()) && getFileExtFromMIME && !observer.tryResumeDownload()) { if (Utils.getConfigBoolean("file.overwrite", false)) { - logger.info("[!] " + Utils.getLocalizedString("deleting.existing.file") + prettySaveAs); + logger.info("[!] " + Utils.getLocalizedString("deleting.existing.file") + " " + prettySaveAs); if (!saveAs.delete()) logger.error("could not delete existing file: " + saveAs.getAbsolutePath()); } else { logger.info("[!] " + Utils.getLocalizedString("skipping") + " " + url + " -- " + Utils.getLocalizedString("file.already.exists") + ": " + prettySaveAs); - observer.downloadExists(url, saveAs.toPath()); + observer.downloadExists(ripUrlId, saveAs.toPath()); return; } } - URL urlToDownload = this.url; boolean redirected = false; int tries = 0; // Number of attempts to download do { tries += 1; try { - logger.info(" Downloading file: " + urlToDownload + (tries > 0 ? " Retry #" + tries : "")); - observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm()); + logger.info(" Downloading file: " + url + (tries > 0 ? " Try #" + tries : "")); + + String urlNoQuery = new URI(url.getProtocol(), url.getAuthority(), url.getPath(), null, null).toURL().toExternalForm(); + observer.sendUpdate(STATUS.DOWNLOAD_STARTED, urlNoQuery); // Setup HTTP request HttpURLConnection huc; - if (this.url.toString().startsWith("https")) { - huc = (HttpsURLConnection) urlToDownload.openConnection(); + if (url.getProtocol().equals("https")) { + huc = (HttpsURLConnection) url.openConnection(); } else { - huc = (HttpURLConnection) urlToDownload.openConnection(); + huc = (HttpURLConnection) url.openConnection(); } huc.setInstanceFollowRedirects(true); // It is important to set both ConnectTimeout and ReadTimeout. If you don't then @@ -140,46 +182,51 @@ public void run() { if (statusCode != 206 && observer.tryResumeDownload() && saveAs.exists()) { // TODO find a better way to handle servers that don't support resuming // downloads then just erroring out - throw new IOException(Utils.getLocalizedString("server.doesnt.support.resuming.downloads")); + observer.downloadErrored(ripUrlId, + Utils.getLocalizedString("server.doesnt.support.resuming.downloads") + ": " + + Utils.getLocalizedString("0.while.downloading.1", statusCode, url.toExternalForm())); + //throw new IOException(Utils.getLocalizedString("server.doesnt.support.resuming.downloads")); + return; } if (statusCode / 100 == 3) { // 3xx Redirect + // FIXME Should not happen because of above line: huc.setInstanceFollowRedirects(true); ??? if (!redirected) { // Don't increment retries on the first redirect tries--; redirected = true; } String location = huc.getHeaderField("Location"); - urlToDownload = new URI(location).toURL(); - // Throw exception so download can be retried - throw new IOException("Redirect status code " + statusCode + " - redirect to " + location); + url = new URI(location).toURL(); // TODO fix redirect with TokenedUrlGetter + logger.debug("Redirect status code {} - redirect to {}", statusCode, location); + continue; // retry } if (statusCode / 100 == 4) { // 4xx errors logger.error("[!] " + Utils.getLocalizedString("nonretriable.status.code") + " " + statusCode + " while downloading from " + url); - observer.downloadErrored(url, Utils.getLocalizedString("nonretriable.status.code") + " " - + statusCode + " while downloading " + url.toExternalForm()); + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("nonretriable.status.code") + " " + + Utils.getLocalizedString("0.while.downloading.1", statusCode, url.toExternalForm())); return; // Not retriable, drop out. } if (statusCode / 100 == 5) { // 5xx errors - observer.downloadErrored(url, Utils.getLocalizedString("retriable.status.code") + " " + statusCode - + " while downloading " + url.toExternalForm()); + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("retriable.status.code") + " " + + Utils.getLocalizedString("0.while.downloading.1", statusCode, url.toExternalForm())); // Throw exception so download can be retried throw new IOException(Utils.getLocalizedString("retriable.status.code") + " " + statusCode); } - if (huc.getContentLength() == 503 && urlToDownload.getHost().endsWith("imgur.com")) { + if (huc.getContentLength() == 503 && url.getHost().endsWith("imgur.com")) { // Imgur image with 503 bytes is "404" logger.error("[!] Imgur image is 404 (503 bytes long): " + url); - observer.downloadErrored(url, "Imgur image is 404: " + url.toExternalForm()); + observer.downloadErrored(ripUrlId, "Imgur image is 404: " + url.toExternalForm()); return; } // If the ripper is using the bytes progress bar set bytesTotal to // huc.getContentLength() if (observer.useByteProgessBar()) { - bytesTotal = huc.getContentLength(); + bytesTotal = huc.getContentLengthLong(); observer.setBytesTotal(bytesTotal); observer.sendUpdate(STATUS.TOTAL_BYTES, bytesTotal); - logger.debug("Size of file at " + this.url + " = " + bytesTotal + "b"); + logger.debug("Size of file at " + url + " = " + bytesTotal + "b"); } // Save file @@ -250,18 +297,25 @@ public void run() { if (shouldSkipFileDownload) { logger.debug("Not downloading whole file because it is over 10mb and this is a test"); } else { + long lastProgressUpdate = 0; + long bytesSinceLastProgressUpdate = 0; while ((bytesRead = bis.read(data)) != -1) { - try { - observer.stopCheck(); - } catch (IOException e) { - observer.downloadErrored(url, Utils.getLocalizedString("download.interrupted")); + if (observer.isPanicked()) { + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("download.interrupted")); return; } fos.write(data, 0, bytesRead); - if (observer.useByteProgessBar()) { - bytesDownloaded += bytesRead; - observer.setBytesCompleted(bytesDownloaded); - observer.sendUpdate(STATUS.COMPLETED_BYTES, bytesDownloaded); + bytesSinceLastProgressUpdate += bytesRead; + long now = System.currentTimeMillis(); + if (now > lastProgressUpdate + 200) { + lastProgressUpdate = now; + observer.sendUpdate(STATUS.CHUNK_BYTES, bytesSinceLastProgressUpdate); + bytesSinceLastProgressUpdate = 0; + if (observer.useByteProgessBar()) { + bytesDownloaded += bytesRead; + observer.setBytesCompleted(bytesDownloaded); + observer.sendUpdate(STATUS.COMPLETED_BYTES, bytesDownloaded); + } } } } @@ -275,20 +329,33 @@ public void run() { break; } catch (HttpStatusException hse) { logger.debug(Utils.getLocalizedString("http.status.exception"), hse); - logger.error("[!] HTTP status " + hse.getStatusCode() + " while downloading from " + urlToDownload); + logger.error("[!] HTTP status " + hse.getStatusCode() + " while downloading from " + hse.getUrl()); if (hse.getStatusCode() == 404 && Utils.getConfigBoolean("errors.skip404", false)) { - observer.downloadErrored(url, - "HTTP status code " + hse.getStatusCode() + " while downloading " + url.toExternalForm()); + observer.downloadErrored(ripUrlId, + Utils.getLocalizedString("0.while.downloading.1", "HTTP " + hse.getStatusCode(), url.toExternalForm())); + return; + } + } catch (IOException e) { + if (guessIsENOSPC(e, saveAs)) { + logger.debug("IOException", e); + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("no.space.left.on.device")); // TODO cancel all rips return; } - } catch (IOException | URISyntaxException e) { logger.debug("IOException", e); logger.error("[!] " + Utils.getLocalizedString("exception.while.downloading.file") + ": " + url + " - " + e.getMessage()); + observer.downloadErrored(ripUrlId, e.getMessage()); + return; + } catch (URISyntaxException e) { + logger.debug("IOException", e); + logger.error("[!] " + Utils.getLocalizedString("exception.while.downloading.file") + ": " + url + " - " + + e.getMessage()); + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("exception.while.downloading.file")); + return; } catch (NullPointerException npe){ logger.error("[!] " + Utils.getLocalizedString("failed.to.download") + " for URL " + url); - observer.downloadErrored(url, + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("failed.to.download") + " " + url.toExternalForm()); return; @@ -296,7 +363,7 @@ public void run() { if (tries > this.retries) { logger.error("[!] " + Utils.getLocalizedString("exceeded.maximum.retries") + " (" + this.retries + ") for URL " + url); - observer.downloadErrored(url, + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("failed.to.download") + " " + url.toExternalForm()); return; } else { @@ -304,9 +371,42 @@ public void run() { Utils.sleep(retrySleep); } } + + // get fresh URL for the next attempt + try { + url = tokenedUrlGetter.getTokenedUrl(); + } catch (HttpStatusException e) { + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("failed.to.get.url.for.0", ripUrlId)); + logger.error("[!] Failed to get URL for " + ripUrlId); + return; // do not retry + } catch (IOException | URISyntaxException e) { + logger.error("[!] Failed to get URL for " + ripUrlId, e); + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("failed.to.get.url.for.0", ripUrlId)); + return; // do not retry + } + } while (true); - observer.downloadCompleted(url, saveAs.toPath()); - logger.info("[+] Saved " + url + " as " + this.prettySaveAs); + observer.downloadCompleted(ripUrlId, saveAs.toPath()); + logger.info("[+] Saved " + url + " as " + prettySaveAs); + } + + @SuppressWarnings("UnnecessaryLocalVariable") + private boolean guessIsENOSPC(IOException e, File saveAs) { + // The ENOSPC IOException message is localized in Java, so this only works on English locale systems. + if (e.getMessage().matches("No space left on device")) { + return true; + } + // Fallback: check usable space on the filesystem + try { + FileStore fs = Files.getFileStore(saveAs.toPath()); + // could check for 0 bytes, but 256 kilobytes is small enough + int downloadBufferSizeBytes = 1024 * 256; + boolean notEnoughUsableBytes = fs.getUsableSpace() < downloadBufferSizeBytes; + return notEnoughUsableBytes; + } catch (IOException ex) { + // unable to determine if no space left on device; fall through + } + return false; } } diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java b/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java index 8ae43743f..08aa615c1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java @@ -1,8 +1,12 @@ package com.rarchives.ripme.ripper; +import java.net.URL; import java.util.concurrent.Executors; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; import com.rarchives.ripme.utils.Utils; import org.apache.logging.log4j.LogManager; @@ -15,42 +19,108 @@ public class DownloadThreadPool { private static final Logger logger = LogManager.getLogger(DownloadThreadPool.class); private ThreadPoolExecutor threadPool = null; - - public DownloadThreadPool() { - initialize("Main"); - } + private final AtomicLong scheduledThreadCount = new AtomicLong(0); + private final String name; public DownloadThreadPool(String threadPoolName) { - initialize(threadPoolName); - } - - /** - * Initializes the threadpool. - * @param threadPoolName Name of the threadpool. - */ - private void initialize(String threadPoolName) { int threads = Utils.getConfigInteger("threads.size", 10); logger.debug("Initializing " + threadPoolName + " thread pool with " + threads + " threads"); - threadPool = (ThreadPoolExecutor) Executors.newFixedThreadPool(threads); + this.name = threadPoolName; + this.threadPool = (ThreadPoolExecutor) Executors.newFixedThreadPool(threads, Thread.ofVirtual().factory()); } + /** * For adding threads to execution pool. * @param t * Thread to be added. */ public void addThread(Runnable t) { + logger.trace("addThread called; name: {}, scheduledThreadCount: {}", name, scheduledThreadCount); + scheduledThreadCount.incrementAndGet(); threadPool.execute(t); } + public void setThreadPoolSize(int threads) { + logger.debug("Setting thread pool size to {}", threads); + if (threads > threadPool.getMaximumPoolSize()) { + threadPool.setMaximumPoolSize(threads); + threadPool.setCorePoolSize(threads); + } else { + threadPool.setCorePoolSize(threads); + threadPool.setMaximumPoolSize(threads); + } + } + + /** + * Tries to shutdown threadpool. + */ + public void waitForThreads(Supplier isFinishedQueueing, URL url) { + logger.trace("waitForThreads called; name: {}; url: {}", name, url); + while (!isFinishedQueueing.get()) { + logger.trace("waiting for items to finish queueing; name: {}; url: {}", name, url); + try { + Thread.sleep(200); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.trace("sleep interrupted; name: {}; url: {}", name, url); + break; + } + } + + logger.trace("about to shutdown thread pool. name: {}; url: {}", name, url); + threadPool.shutdown(); + logger.trace("thread pool shutdown. name: {}; url: {}", name, url); + try { + threadPool.awaitTermination(3600, TimeUnit.SECONDS); + logger.trace("thread pool terminated. name: {}; url: {}", name, url); + } catch (InterruptedException e) { + logger.error("[!] Interrupted while waiting for threads to finish: ", e); + } + } + /** * Tries to shutdown threadpool. */ - public void waitForThreads() { + public void waitForThreads(int expectedScheduledThreads, AtomicBoolean shouldStop, URL url) { + logger.trace("waitForThreads called; name: {}; url: {}", name, url); + while (getScheduledThreadCount() < expectedScheduledThreads && !shouldStop.get()) { + logger.trace("waiting for scheduled threads to equal expected scheduled threads; name: {}; scheduled: {}; expected: {} url: {}", name, scheduledThreadCount, expectedScheduledThreads, url); + try { + Thread.sleep(200); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + + logger.trace("about to shutdown thread pool. name: {}; url: {}", name, url); threadPool.shutdown(); + logger.trace("thread pool shutdown. name: {}; url: {}", name, url); try { threadPool.awaitTermination(3600, TimeUnit.SECONDS); + logger.trace("thread pool terminated. name: {}; url: {}", name, url); } catch (InterruptedException e) { logger.error("[!] Interrupted while waiting for threads to finish: ", e); } } + + public int getPendingThreadCount() { + return threadPool.getQueue().size(); + } + + /** + * @return The approximate active thread count + */ + public int getActiveThreadCount() { + return threadPool.getActiveCount(); + } + + public long getCompletedThreadCount() { + return threadPool.getCompletedTaskCount(); + } + + public long getScheduledThreadCount() { + //return threadPool.getTaskCount(); // approximate, bad + return scheduledThreadCount.get(); + } } diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java deleted file mode 100644 index 9430adce3..000000000 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java +++ /dev/null @@ -1,160 +0,0 @@ -package com.rarchives.ripme.ripper; - -import java.io.BufferedInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.HttpURLConnection; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; - -import javax.net.ssl.HttpsURLConnection; - -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; -import com.rarchives.ripme.utils.Utils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -/** - * Thread for downloading files. - * Includes retry logic, observer notifications, and other goodies. - */ -class DownloadVideoThread implements Runnable { - - private static final Logger logger = LogManager.getLogger(DownloadVideoThread.class); - - private final URL url; - private final Path saveAs; - private final String prettySaveAs; - private final AbstractRipper observer; - private final int retries; - - public DownloadVideoThread(URL url, Path saveAs, AbstractRipper observer) { - super(); - this.url = url; - this.saveAs = saveAs; - this.prettySaveAs = Utils.removeCWD(saveAs); - this.observer = observer; - this.retries = Utils.getConfigInteger("download.retries", 1); - } - - /** - * Attempts to download the file. Retries as needed. - * Notifies observers upon completion/error/warn. - */ - @Override - public void run() { - try { - observer.stopCheck(); - } catch (IOException e) { - observer.downloadErrored(url, "Download interrupted"); - return; - } - if (Files.exists(saveAs)) { - if (Utils.getConfigBoolean("file.overwrite", false)) { - logger.info("[!] Deleting existing file" + prettySaveAs); - try { - Files.delete(saveAs); - } catch (IOException e) { - e.printStackTrace(); - } - } else { - logger.info("[!] Skipping " + url + " -- file already exists: " + prettySaveAs); - observer.downloadExists(url, saveAs); - return; - } - } - - int bytesTotal, bytesDownloaded = 0; - try { - bytesTotal = getTotalBytes(this.url); - } catch (IOException e) { - logger.error("Failed to get file size at " + this.url, e); - observer.downloadErrored(this.url, "Failed to get file size of " + this.url); - return; - } - observer.setBytesTotal(bytesTotal); - observer.sendUpdate(STATUS.TOTAL_BYTES, bytesTotal); - logger.debug("Size of file at " + this.url + " = " + bytesTotal + "b"); - - int tries = 0; // Number of attempts to download - do { - InputStream bis = null; OutputStream fos = null; - byte[] data = new byte[1024 * 256]; - int bytesRead; - try { - logger.info(" Downloading file: " + url + (tries > 0 ? " Retry #" + tries : "")); - observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm()); - - // Setup HTTP request - HttpURLConnection huc; - if (this.url.toString().startsWith("https")) { - huc = (HttpsURLConnection) this.url.openConnection(); - } - else { - huc = (HttpURLConnection) this.url.openConnection(); - } - huc.setInstanceFollowRedirects(true); - huc.setConnectTimeout(0); // Never timeout - huc.setRequestProperty("accept", "*/*"); - huc.setRequestProperty("Referer", this.url.toExternalForm()); // Sic - huc.setRequestProperty("User-agent", AbstractRipper.USER_AGENT); - tries += 1; - logger.debug("Request properties: " + huc.getRequestProperties().toString()); - huc.connect(); - // Check status code - bis = new BufferedInputStream(huc.getInputStream()); - fos = Files.newOutputStream(saveAs); - while ( (bytesRead = bis.read(data)) != -1) { - try { - observer.stopCheck(); - } catch (IOException e) { - observer.downloadErrored(url, "Download interrupted"); - return; - } - fos.write(data, 0, bytesRead); - bytesDownloaded += bytesRead; - observer.setBytesCompleted(bytesDownloaded); - observer.sendUpdate(STATUS.COMPLETED_BYTES, bytesDownloaded); - } - bis.close(); - fos.close(); - break; // Download successful: break out of infinite loop - } catch (IOException e) { - logger.error("[!] Exception while downloading file: " + url + " - " + e.getMessage(), e); - } finally { - // Close any open streams - try { - if (bis != null) { bis.close(); } - } catch (IOException ignored) { } - try { - if (fos != null) { fos.close(); } - } catch (IOException ignored) { } - } - if (tries > this.retries) { - logger.error("[!] Exceeded maximum retries (" + this.retries + ") for URL " + url); - observer.downloadErrored(url, "Failed to download " + url.toExternalForm()); - return; - } - } while (true); - observer.downloadCompleted(url, saveAs); - logger.info("[+] Saved " + url + " as " + this.prettySaveAs); - } - - /** - * @param url - * Target URL - * @return - * Returns connection length - */ - private int getTotalBytes(URL url) throws IOException { - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - conn.setRequestMethod("HEAD"); - conn.setRequestProperty("accept", "*/*"); - conn.setRequestProperty("Referer", this.url.toExternalForm()); // Sic - conn.setRequestProperty("User-agent", AbstractRipper.USER_AGENT); - return conn.getContentLength(); - } - -} \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/RipUrlId.java b/src/main/java/com/rarchives/ripme/ripper/RipUrlId.java new file mode 100644 index 000000000..9e376bb2e --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/RipUrlId.java @@ -0,0 +1,97 @@ +package com.rarchives.ripme.ripper; + +import java.net.URL; +import java.util.Objects; + +/** + * RipUrlId represents a unique file on a host. + * Necessary because some files may be accessible from multiple URLs, for example: + * - a file in multiple albums, or + * - a file only accessible with a tokened URL. + */ +public class RipUrlId { + Class ripper; + String ripperHost; + String ripUrlId; + URL url; + + /** + * @param ripper The ripper associated with the id + * @param ripperHost The ripper's getHost(), because a ripper may support multiple hosts + * @param ripUrlId The unique identifier of the file fetchable by the ripper + */ + public RipUrlId(Class ripper, String ripperHost, String ripUrlId) { + if (ripper == null) { + throw new IllegalArgumentException("ripper cannot be null"); + } + if (ripperHost == null) { + throw new IllegalArgumentException("ripperHost cannot be null"); + } + if (ripUrlId == null) { + throw new IllegalArgumentException("ripUrlId cannot be null"); + } + this.ripper = ripper; + this.ripperHost = ripperHost; + this.ripUrlId = ripUrlId; + } + + /** + * Transitionary constructor for rippers that do not yet create an id + * + * @param ripper The ripper associated with the id + * @param ripperHost The ripper's getHost(), because a ripper may support multiple hosts + * @param url A URL fetchable by the ripper + * @deprecated The other constructor is preferable + */ + @Deprecated + public RipUrlId(Class ripper, String ripperHost, URL url) { + if (ripper == null) { + throw new IllegalArgumentException("ripper cannot be null"); + } + if (ripperHost == null) { + throw new IllegalArgumentException("ripperHost cannot be null"); + } + if (url == null) { + throw new IllegalArgumentException("url cannot be null"); + } + this.ripper = ripper; + this.ripperHost = ripperHost; + this.url = url; + } + + public Class getRipper() { + return ripper; + } + + public String getRipperHost() { + return ripperHost; + } + + public String getRipUrlId() { + return ripUrlId; + } + + public URL getUrl() { + return url; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + RipUrlId ripUrlId1 = (RipUrlId) o; + return Objects.equals(ripper, ripUrlId1.ripper) && Objects.equals(ripperHost, ripUrlId1.ripperHost) && Objects.equals(ripUrlId, ripUrlId1.ripUrlId) && Objects.equals(url, ripUrlId1.url); + } + + @Override + public int hashCode() { + return Objects.hash(ripper, ripperHost, ripUrlId, url); + } + + @Override + public String toString() { + if (url != null) { + return url.toString(); + } + return ripper.getSimpleName() + ": " + ripperHost + ": " + ripUrlId; + } +} diff --git a/src/main/java/com/rarchives/ripme/ripper/TokenedUrlGetter.java b/src/main/java/com/rarchives/ripme/ripper/TokenedUrlGetter.java new file mode 100644 index 000000000..706c7bc72 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/TokenedUrlGetter.java @@ -0,0 +1,14 @@ +package com.rarchives.ripme.ripper; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; + +public interface TokenedUrlGetter { + /** + * @return The URL of the file to fetch + * @throws IOException May be thrown if a tokened URI can't be fetched + * @throws URISyntaxException May be thrown if a URI can't be constructed + */ + URL getTokenedUrl() throws IOException, URISyntaxException; +} diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index 785f3d92b..bc57b9c4c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -12,17 +12,12 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import com.rarchives.ripme.ui.RipStatusMessage; -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Utils; public abstract class VideoRipper extends AbstractRipper { private static final Logger logger = LogManager.getLogger(VideoRipper.class); - private int bytesTotal = 1; - private int bytesCompleted = 1; - protected VideoRipper(URL url) throws IOException { super(url); } @@ -33,57 +28,19 @@ protected VideoRipper(URL url) throws IOException { public abstract String getGID(URL url) throws MalformedURLException; - @Override - public void setBytesTotal(int bytes) { - this.bytesTotal = bytes; - } - - @Override - public void setBytesCompleted(int bytes) { - this.bytesCompleted = bytes; - } - @Override public String getAlbumTitle(URL url) { return "videos"; } @Override - public boolean addURLToDownload(URL url, Path saveAs) { - if (Utils.getConfigBoolean("urls_only.save", false)) { - // Output URL to file - String urlFile = this.workingDir + "/urls.txt"; - - try (FileWriter fw = new FileWriter(urlFile, true)) { - fw.write(url.toExternalForm()); - fw.write("\n"); - - RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, urlFile); - observer.update(this, msg); - } catch (IOException e) { - logger.error("Error while writing to " + urlFile, e); - return false; - } - } else { - if (isThisATest()) { - // Tests shouldn't download the whole video - // Just change this.url to the download URL so the test knows we found it. - logger.debug("Test rip, found URL: " + url); - this.url = url; - return true; - } - if (shouldIgnoreURL(url)) { - sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); - return false; - } - threadPool.addThread(new DownloadVideoThread(url, saveAs, this)); - } + public boolean useByteProgessBar() { return true; } @Override - public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { - return addURLToDownload(url, saveAs); + protected boolean allowDuplicates() { + return false; } /** @@ -120,74 +77,6 @@ public int getCompletionPercentage() { return (int) (100 * (bytesCompleted / (float) bytesTotal)); } - /** - * Runs if download successfully completed. - * - * @param url Target URL - * @param saveAs Path to file, including filename. - */ - @Override - public void downloadCompleted(URL url, Path saveAs) { - if (observer == null) { - return; - } - - try { - String path = Utils.removeCWD(saveAs); - RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); - observer.update(this, msg); - - checkIfComplete(); - } catch (Exception e) { - logger.error("Exception while updating observer: ", e); - } - } - - /** - * Runs if the download errored somewhere. - * - * @param url Target URL - * @param reason Reason why the download failed. - */ - @Override - public void downloadErrored(URL url, String reason) { - if (observer == null) { - return; - } - - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason)); - checkIfComplete(); - } - - /** - * Runs if user tries to redownload an already existing File. - * @param url Target URL - * @param file Existing file - */ - @Override - public void downloadExists(URL url, Path file) { - if (observer == null) { - return; - } - - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " already saved as " + file)); - checkIfComplete(); - } - - /** - * Gets the status and changes it to a human-readable form. - * - * @return Status of current download. - */ - @Override - public String getStatusText() { - return String.valueOf(getCompletionPercentage()) + - "% - " + - Utils.bytesToHumanReadable(bytesCompleted) + - " / " + - Utils.bytesToHumanReadable(bytesTotal); - } - /** * Sanitizes URL. * Usually just returns itself. @@ -197,18 +86,4 @@ public URL sanitizeURL(URL url) throws MalformedURLException { return url; } - /** - * Notifies observers and updates state if all files have been ripped. - */ - @Override - protected void checkIfComplete() { - if (observer == null) { - return; - } - - if (bytesCompleted >= bytesTotal) { - super.checkIfComplete(); - } - } - } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java index 98510250c..4bb197e77 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java @@ -30,7 +30,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; @@ -94,7 +93,6 @@ public class DeviantartRipper extends AbstractHTMLRipper { private boolean usingCatPath = false; private int downloadCount = 0; private Map cookies = new HashMap(); - private DownloadThreadPool deviantartThreadPool = new DownloadThreadPool("deviantart"); private ArrayList names = new ArrayList(); List allowedCookies = Arrays.asList("agegate_state", "userinfo", "auth", "auth_secure"); @@ -106,11 +104,6 @@ public class DeviantartRipper extends AbstractHTMLRipper { private final String userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0"; private final String utilsKey = "DeviantartLogin.cookies"; //for config file - @Override - public DownloadThreadPool getThreadPool() { - return deviantartThreadPool; - } - public DeviantartRipper(URL url) throws IOException { super(url); } @@ -304,7 +297,7 @@ protected void downloadURL(URL url, int index) { // Start Thread and add to pool. DeviantartImageThread t = new DeviantartImageThread(url); - deviantartThreadPool.addThread(t); + getCrawlerThreadPool().addThread(t); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java index 9b40f0542..d0fbbbb9e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java @@ -19,7 +19,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Http; @@ -37,8 +36,6 @@ public class E621Ripper extends AbstractHTMLRipper { private static Pattern gidPatternNew = null; private static Pattern gidPatternPoolNew = null; - private DownloadThreadPool e621ThreadPool = new DownloadThreadPool("e621"); - private Map cookies = new HashMap(); private String userAgent = USER_AGENT; @@ -78,11 +75,6 @@ private Document getDocument(String url) throws IOException { return getDocument(url, 1); } - @Override - public DownloadThreadPool getThreadPool() { - return e621ThreadPool; - } - @Override public String getDomain() { return "e621.net"; @@ -136,7 +128,7 @@ public void downloadURL(final URL url, int index) { // rate limit sleep(3000); // addURLToDownload(url, getPrefix(index)); - e621ThreadPool.addThread(new E621FileThread(url, getPrefix(index))); + getCrawlerThreadPool().addThread(new E621FileThread(url, getPrefix(index))); } private String getTerm(URL url) throws MalformedURLException { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java index 5349f55c7..d58bcc4c9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java @@ -21,7 +21,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Http; @@ -44,8 +43,6 @@ public class EHentaiRipper extends AbstractHTMLRipper { } private String lastURL = null; - // Thread pool for finding direct image links from "image" pages (html) - private final DownloadThreadPool ehentaiThreadPool = new DownloadThreadPool("ehentai"); // Current HTML document private Document albumDoc = null; @@ -53,11 +50,6 @@ public EHentaiRipper(URL url) throws IOException { super(url); } - @Override - public DownloadThreadPool getThreadPool() { - return ehentaiThreadPool; - } - @Override public String getHost() { return "e-hentai"; @@ -194,7 +186,7 @@ public List getURLsFromPage(Document page) { @Override public void downloadURL(URL url, int index) { EHentaiImageThread t = new EHentaiImageThread(url, index, this.workingDir.toPath()); - ehentaiThreadPool.addThread(t); + getCrawlerThreadPool().addThread(t); try { Thread.sleep(IMAGE_SLEEP_TIME); } catch (InterruptedException e) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java index 1f1954207..a958b0972 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java @@ -28,8 +28,6 @@ public class FlickrRipper extends AbstractHTMLRipper { private static final Logger logger = LogManager.getLogger(FlickrRipper.class); - private final DownloadThreadPool flickrThreadPool; - private enum UrlType { USER, PHOTOSET @@ -45,11 +43,6 @@ private class Album { } } - @Override - public DownloadThreadPool getThreadPool() { - return flickrThreadPool; - } - @Override public boolean hasASAPRipping() { return true; @@ -57,7 +50,6 @@ public boolean hasASAPRipping() { public FlickrRipper(URL url) throws IOException { super(url); - flickrThreadPool = new DownloadThreadPool(); } @Override @@ -261,6 +253,8 @@ public List getURLsFromPage(Document doc) { } int totalPages = rootData.getInt("pages"); + int totalFiles = rootData.getInt("total"); + setItemsTotal(totalFiles); logger.info(jsonData); JSONArray pictures = rootData.getJSONArray("photo"); for (int i = 0; i < pictures.length(); i++) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java index ad9075d0d..cb7c49479 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java @@ -54,15 +54,6 @@ private void warnAboutSharedAccount(String loginCookies) { } } - // Thread pool for finding direct image links from "image" pages (html) - private DownloadThreadPool furaffinityThreadPool - = new DownloadThreadPool( "furaffinity"); - - @Override - public DownloadThreadPool getThreadPool() { - return furaffinityThreadPool; - } - public FuraffinityRipper(URL url) throws IOException { super(url); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java index 7183f1d79..54a98f306 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java @@ -18,7 +18,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; public class HqpornerRipper extends AbstractHTMLRipper { @@ -30,7 +29,6 @@ public class HqpornerRipper extends AbstractHTMLRipper { private Pattern p1 = Pattern.compile("https?://hqporner.com/hdporn/([a-zA-Z0-9_-]*).html/?$"); // video pattern. private Pattern p2 = Pattern.compile("https://hqporner.com/([a-zA-Z0-9/_-]+)"); // category/top/actress/studio pattern. private Pattern p3 = Pattern.compile("https?://[A-Za-z0-9/.-_]+\\.mp4"); // to match links ending with .mp4 - private DownloadThreadPool hqpornerThreadPool = new DownloadThreadPool("hqpornerThreadPool"); private String subdirectory = ""; public HqpornerRipper(URL url) throws IOException { @@ -111,7 +109,7 @@ public boolean tryResumeDownload() { @Override public void downloadURL(URL url, int index) { - hqpornerThreadPool.addThread(new HqpornerDownloadThread(url, index, subdirectory)); + getCrawlerThreadPool().addThread(new HqpornerDownloadThread(url, index, subdirectory)); } @Override @@ -123,11 +121,6 @@ public Document getNextPage(Document doc) throws IOException { throw new IOException("No next page found."); } - @Override - public DownloadThreadPool getThreadPool() { - return hqpornerThreadPool; - } - @Override public boolean useByteProgessBar() { return true; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java index d3e5b2ce8..104106301 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java @@ -1,7 +1,6 @@ package com.rarchives.ripme.ripper.rippers; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; import java.io.IOException; @@ -29,13 +28,6 @@ public class ImagebamRipper extends AbstractHTMLRipper { private static final Logger logger = LogManager.getLogger(ImagebamRipper.class); - // Thread pool for finding direct image links from "image" pages (html) - private DownloadThreadPool imagebamThreadPool = new DownloadThreadPool("imagebam"); - @Override - public DownloadThreadPool getThreadPool() { - return imagebamThreadPool; - } - public ImagebamRipper(URL url) throws IOException { super(url); } @@ -90,7 +82,7 @@ public List getURLsFromPage(Document doc) { @Override public void downloadURL(URL url, int index) { ImagebamImageThread t = new ImagebamImageThread(url, index); - imagebamThreadPool.addThread(t); + getCrawlerThreadPool().addThread(t); sleep(500); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java index 5421e14dd..c63bd36e9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java @@ -17,7 +17,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; @@ -25,13 +24,6 @@ public class ImagevenueRipper extends AbstractHTMLRipper { private static final Logger logger = LogManager.getLogger(ImagevenueRipper.class); - // Thread pool for finding direct image links from "image" pages (html) - private DownloadThreadPool imagevenueThreadPool = new DownloadThreadPool("imagevenue"); - @Override - public DownloadThreadPool getThreadPool() { - return imagevenueThreadPool; - } - public ImagevenueRipper(URL url) throws IOException { super(url); } @@ -72,7 +64,7 @@ public List getURLsFromPage(Document doc) { public void downloadURL(URL url, int index) { ImagevenueImageThread t = new ImagevenueImageThread(url, index); - imagevenueThreadPool.addThread(t); + getCrawlerThreadPool().addThread(t); } /** diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java index 5bb7b0020..afbab931e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java @@ -205,7 +205,7 @@ public void rip() throws IOException { } catch (URISyntaxException e) { throw new IOException("Failed ripping " + this.url, e); } - waitForThreads(); + waitForRipperThreads(); } private void ripSingleImage(URL url) throws IOException, URISyntaxException { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java index 7157e49dd..2cf37ef6e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java @@ -19,7 +19,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; /** @@ -38,8 +37,6 @@ public class ListalRipper extends AbstractHTMLRipper { private String listId = null; // listId to get more images via POST. private UrlType urlType = UrlType.UNKNOWN; - private DownloadThreadPool listalThreadPool = new DownloadThreadPool("listalThreadPool"); - public ListalRipper(URL url) throws IOException { super(url); } @@ -77,7 +74,7 @@ public List getURLsFromPage(Document page) { @Override public void downloadURL(URL url, int index) { - listalThreadPool.addThread(new ListalImageDownloadThread(url, index)); + getCrawlerThreadPool().addThread(new ListalImageDownloadThread(url, index)); } @Override @@ -137,11 +134,6 @@ public Document getNextPage(Document page) throws IOException, URISyntaxExceptio } - @Override - public DownloadThreadPool getThreadPool() { - return listalThreadPool; - } - /** * Returns the image urls for UrlType LIST. */ diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java index 955e85a34..dc1fa3f67 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java @@ -31,11 +31,8 @@ public class MotherlessRipper extends AbstractHTMLRipper { private static final String DOMAIN = "motherless.com", HOST = "motherless"; - private DownloadThreadPool motherlessThreadPool; - public MotherlessRipper(URL url) throws IOException { super(url); - motherlessThreadPool = new DownloadThreadPool(); } @Override @@ -117,7 +114,7 @@ protected List getURLsFromPage(Document page) { protected void downloadURL(URL url, int index) { // Create thread for finding image at "url" page MotherlessImageRunnable mit = new MotherlessImageRunnable(url, index); - motherlessThreadPool.addThread(mit); + getCrawlerThreadPool().addThread(mit); try { Thread.sleep(IMAGE_SLEEP_TIME); } catch (InterruptedException e) { @@ -155,11 +152,6 @@ public String getGID(URL url) throws MalformedURLException { throw new MalformedURLException("Expected URL format: https://motherless.com/GIXXXXXXX, got: " + url); } - @Override - protected DownloadThreadPool getThreadPool() { - return motherlessThreadPool; - } - /** * Helper class to find and download images found on "image" pages */ diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java index d6b17b02f..e69dbed81 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java @@ -17,7 +17,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; public class NfsfwRipper extends AbstractHTMLRipper { @@ -34,12 +33,8 @@ public class NfsfwRipper extends AbstractHTMLRipper { "https?://[wm.]*nfsfw.com/gallery/v/[^/]+/(.+)$" ); - // threads pool for downloading images from image pages - private DownloadThreadPool nfsfwThreadPool; - public NfsfwRipper(URL url) throws IOException { super(url); - nfsfwThreadPool = new DownloadThreadPool("NFSFW"); } @Override @@ -105,7 +100,7 @@ protected void downloadURL(URL url, int index) { index = ++this.index; } NfsfwImageThread t = new NfsfwImageThread(url, currentDir, index); - nfsfwThreadPool.addThread(t); + getCrawlerThreadPool().addThread(t); } @Override @@ -141,11 +136,6 @@ public String getGID(URL url) throws MalformedURLException { + " Got: " + url); } - @Override - public DownloadThreadPool getThreadPool() { - return nfsfwThreadPool; - } - @Override public boolean hasQueueSupport() { return true; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java index 41693a3ee..13df2aa74 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java @@ -27,9 +27,6 @@ public class NhentaiRipper extends AbstractHTMLRipper { private Document firstPage; - // Thread pool for finding direct image links from "image" pages (html) - private DownloadThreadPool nhentaiThreadPool = new DownloadThreadPool("nhentai"); - @Override public boolean hasQueueSupport() { return true; @@ -51,11 +48,6 @@ public List getAlbumsToQueue(Document doc) { return urlsToAddToQueue; } - @Override - public DownloadThreadPool getThreadPool() { - return nhentaiThreadPool; - } - public NhentaiRipper(URL url) throws IOException { super(url); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java index 481ab1ede..c6279c3e0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java @@ -18,7 +18,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; @@ -31,9 +30,6 @@ public class PornhubRipper extends AbstractHTMLRipper { private static final String DOMAIN = "pornhub.com", HOST = "Pornhub"; - // Thread pool for finding direct image links from "image" pages (html) - private DownloadThreadPool pornhubThreadPool = new DownloadThreadPool("pornhub"); - public PornhubRipper(URL url) throws IOException { super(url); } @@ -82,7 +78,7 @@ protected List getURLsFromPage(Document page) { @Override protected void downloadURL(URL url, int index) { PornhubImageThread t = new PornhubImageThread(url, index, this.workingDir.toPath()); - pornhubThreadPool.addThread(t); + getCrawlerThreadPool().addThread(t); try { Thread.sleep(IMAGE_SLEEP_TIME); } catch (InterruptedException e) { @@ -119,11 +115,6 @@ public String getGID(URL url) throws MalformedURLException { + " Got: " + url); } - @Override - public DownloadThreadPool getThreadPool(){ - return pornhubThreadPool; - } - public boolean canRip(URL url) { return url.getHost().endsWith(DOMAIN) && url.getPath().startsWith("/album"); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index 2419cd035..cb6b8eeb7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -114,7 +114,7 @@ public void rip() throws IOException { } catch (URISyntaxException e) { new IOException(e.getMessage()); } - waitForThreads(); + waitForRipperThreads(); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java index 6bbda8757..7e92b103b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java @@ -236,7 +236,7 @@ public void rip() throws IOException { } } - waitForThreads(); + waitForRipperThreads(); } private boolean handleJSON(JSONObject json) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java index b6b39bc61..5b71081d5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java @@ -158,7 +158,7 @@ public void rip() throws IOException, URISyntaxException { for (int index = 0; index < URLs.size(); index ++) { downloadURL(new URI(URLs.get(index)).toURL(), index); } - waitForThreads(); + waitForRipperThreads(); } else { RIP_TYPE = RipType.IMAGE; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/CliphunterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/CliphunterRipper.java index 1e7a48f8e..76527ceea 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/CliphunterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/CliphunterRipper.java @@ -78,6 +78,6 @@ public void rip() throws IOException, URISyntaxException { } } addURLToDownload(new URI(vidURL).toURL(), HOST + "_" + getGID(this.url)); - waitForThreads(); + waitForRipperThreads(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/MotherlessVideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/MotherlessVideoRipper.java index 0f95aaafc..c6cf3f123 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/MotherlessVideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/MotherlessVideoRipper.java @@ -70,6 +70,6 @@ public void rip() throws IOException, URISyntaxException { } String vidUrl = vidUrls.get(0); addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url)); - waitForThreads(); + waitForRipperThreads(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java index aa8b90541..6f10ec429 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java @@ -154,6 +154,6 @@ public void rip() throws IOException, URISyntaxException { } addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + bestQuality + "p_" + getGID(this.url)); - waitForThreads(); + waitForRipperThreads(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java index 076e90ca6..f6249e9d0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java @@ -75,6 +75,6 @@ public void rip() throws IOException, URISyntaxException { addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + title); } } - waitForThreads(); + waitForRipperThreads(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/ViddmeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/ViddmeRipper.java index a2cff267d..31c127d39 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/ViddmeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/ViddmeRipper.java @@ -68,6 +68,6 @@ public void rip() throws IOException, URISyntaxException { String vidUrl = videos.first().attr("content"); vidUrl = vidUrl.replaceAll("&", "&"); addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url)); - waitForThreads(); + waitForRipperThreads(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java index 00e77c427..4829cff86 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java @@ -63,6 +63,6 @@ public void rip() throws IOException, URISyntaxException { } String vidUrl = mp4s.get(0); addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url)); - waitForThreads(); + waitForRipperThreads(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java index 4a7ea8ccd..1ab5c1ef4 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java @@ -61,7 +61,7 @@ public void rip() throws IOException, URISyntaxException { logger.info(" Retrieving " + this.url); String videoURL = getVideoURLAtPage(this.url.toExternalForm()); addURLToDownload(new URI(videoURL).toURL(), HOST + "_" + getGID(this.url)); - waitForThreads(); + waitForRipperThreads(); } public static String getVideoURLAtPage(String url) throws IOException { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/YuvutuRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/YuvutuRipper.java index 2fe0291e6..61762f3bb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/YuvutuRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/YuvutuRipper.java @@ -77,6 +77,6 @@ public void rip() throws IOException, URISyntaxException { addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url)); } } - waitForThreads(); + waitForRipperThreads(); } } diff --git a/src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java b/src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java index 8d3fc1af7..9705de6f3 100644 --- a/src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java @@ -47,6 +47,7 @@ public static String getClipboardString() { } class AutoripThread extends Thread { + private static final Pattern rippableUrlPattern = Pattern.compile("^(https?|ftp|file)://[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]"); volatile boolean isRunning = false; private final Set rippedURLs = new HashSet<>(); @@ -57,8 +58,7 @@ public void run() { // Check clipboard String clipboard = ClipboardUtils.getClipboardString(); if (clipboard != null) { - Pattern p = Pattern.compile("^(https?|ftp|file)://[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]"); - Matcher m = p.matcher(clipboard); + Matcher m = rippableUrlPattern.matcher(clipboard); while (m.find()) { String url = m.group(); if (!rippedURLs.contains(url)) { @@ -68,7 +68,7 @@ public void run() { } } } - Thread.sleep(1000); + Thread.sleep(250); } } catch (InterruptedException e) { e.printStackTrace(); diff --git a/src/main/java/com/rarchives/ripme/ui/DeselectableButtonGroup.java b/src/main/java/com/rarchives/ripme/ui/DeselectableButtonGroup.java new file mode 100644 index 000000000..51ae63c0a --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ui/DeselectableButtonGroup.java @@ -0,0 +1,14 @@ +package com.rarchives.ripme.ui; + +import javax.swing.*; + +public class DeselectableButtonGroup extends ButtonGroup { + @Override + public void setSelected(ButtonModel model, boolean selected) { + if (selected) { + super.setSelected(model, selected); + } else { + clearSelection(); + } + } +} diff --git a/src/main/java/com/rarchives/ripme/ui/History.java b/src/main/java/com/rarchives/ripme/ui/History.java index 190eeeb8e..735a133ac 100644 --- a/src/main/java/com/rarchives/ripme/ui/History.java +++ b/src/main/java/com/rarchives/ripme/ui/History.java @@ -19,6 +19,7 @@ public class History { private final List list; private static final String[] COLUMNS = new String[] { "URL", + "N", "created", "modified", "#", @@ -52,20 +53,15 @@ public int getColumnCount() { } public Object getValueAt(int row, int col) { HistoryEntry entry = this.list.get(row); - switch (col) { - case 0: - return entry.url; - case 1: - return dateToHumanReadable(entry.startDate); - case 2: - return dateToHumanReadable(entry.modifiedDate); - case 3: - return entry.count; - case 4: - return entry.selected; - default: - return null; - } + return switch (col) { + case 0 -> entry.url; + case 1 -> row; + case 2 -> dateToHumanReadable(entry.startDate); + case 3 -> dateToHumanReadable(entry.modifiedDate); + case 4 -> entry.count; + case 5 -> entry.selected; + default -> null; + }; } private String dateToHumanReadable(Date date) { SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd"); diff --git a/src/main/java/com/rarchives/ripme/ui/HistoryMenuMouseListener.java b/src/main/java/com/rarchives/ripme/ui/HistoryMenuMouseListener.java index 8a69477cc..0f6045e59 100644 --- a/src/main/java/com/rarchives/ripme/ui/HistoryMenuMouseListener.java +++ b/src/main/java/com/rarchives/ripme/ui/HistoryMenuMouseListener.java @@ -15,6 +15,7 @@ class HistoryMenuMouseListener extends MouseAdapter { private JPopupMenu popup = new JPopupMenu(); private JTable tableComponent; + private final int checkboxColumn = 5; @SuppressWarnings("serial") public HistoryMenuMouseListener() { @@ -22,7 +23,7 @@ public HistoryMenuMouseListener() { @Override public void actionPerformed(ActionEvent ae) { for (int row = 0; row < tableComponent.getRowCount(); row++) { - tableComponent.setValueAt(true, row, 4); + tableComponent.setValueAt(true, row, checkboxColumn); } } }; @@ -32,7 +33,7 @@ public void actionPerformed(ActionEvent ae) { @Override public void actionPerformed(ActionEvent ae) { for (int row = 0; row < tableComponent.getRowCount(); row++) { - tableComponent.setValueAt(false, row, 4); + tableComponent.setValueAt(false, row, checkboxColumn); } } }; @@ -44,7 +45,7 @@ public void actionPerformed(ActionEvent ae) { @Override public void actionPerformed(ActionEvent ae) { for (int row : tableComponent.getSelectedRows()) { - tableComponent.setValueAt(true, row, 4); + tableComponent.setValueAt(true, row, checkboxColumn); } } }; @@ -54,7 +55,7 @@ public void actionPerformed(ActionEvent ae) { @Override public void actionPerformed(ActionEvent ae) { for (int row : tableComponent.getSelectedRows()) { - tableComponent.setValueAt(false, row, 4); + tableComponent.setValueAt(false, row, checkboxColumn); } } }; diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 35357830c..5566079d9 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -15,31 +15,37 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Collections; -import java.util.Date; +import java.util.*; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; import java.util.stream.Stream; import javax.imageio.ImageIO; import javax.swing.*; +import javax.swing.border.Border; import javax.swing.border.EmptyBorder; import javax.swing.event.DocumentEvent; import javax.swing.event.DocumentListener; import javax.swing.event.ListDataEvent; import javax.swing.event.ListDataListener; import javax.swing.table.AbstractTableModel; +import javax.swing.table.TableCellRenderer; import javax.swing.text.*; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.core.LoggerContext; -import org.apache.logging.log4j.core.config.Configuration; -import org.apache.logging.log4j.core.config.LoggerConfig; import com.rarchives.ripme.ripper.AbstractRipper; import com.rarchives.ripme.uiUtils.ContextActionProtections; +import com.rarchives.ripme.utils.DebouncedRunnable; import com.rarchives.ripme.utils.RipUtils; +import com.rarchives.ripme.utils.TransferRate; import com.rarchives.ripme.utils.Utils; /** @@ -48,30 +54,45 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static final Logger LOGGER = LogManager.getLogger(MainWindow.class); - - /* not static! */ - private boolean isRipping = false; // Flag to indicate if we're ripping something + private static final String RIPME_PANEL = "ripme.panel"; private static JFrame mainFrame; private static JTextField ripTextfield; private static JButton ripButton, stopButton; + private static JButton panicButton; private static JLabel statusLabel; - private static JButton openButton; - private static JProgressBar statusProgress; + private static final ProgressTextField currentlyRippingProgress = new ProgressTextField(); + private static final JLabel pendingValue = new MinimumWidthLabel("1000", "0"); + private static final JLabel pendingLabel = new JLabel("Pending"); + private static final JLabel activeValue = new MinimumWidthLabel("1000", "0"); + private static final JLabel activeLabel = new JLabel("Active"); + private static final JLabel completedValue = new MinimumWidthLabel("1000", "0"); + private static final JLabel completedLabel = new JLabel("Completed"); + private static final JLabel erroredValue = new MinimumWidthLabel("1000", "0"); + private static final JLabel erroredLabel = new JLabel("Errored"); + private static final JLabel totalValue = new MinimumWidthLabel("1000", "0"); + private static final JLabel totalLabel = new JLabel("Total"); + private static final JLabel transferRateValue = new MinimumWidthLabel("999.00 KiB/s", "0.00 B/s"); + private static final JLabel transferRateLabel = new JLabel("Speed"); + private static final JButton openButton = new JButton(); // Put an empty JPanel on the bottom of the window to keep components // anchored to the top when there is no open lower panel private static JPanel emptyPanel; + private static final ButtonGroup panelButtonGroup = new DeselectableButtonGroup(); + // Log - private static JButton optionLog; + private static JToggleButton optionLog; private static JPanel logPanel; private static JTextPane logText; + private static final Queue logLineLengths = new LinkedList<>(); + private static final int MAX_LOG_PANE_LINES = 1000; // History - private static JButton optionHistory; + private static JToggleButton optionHistory; private static final History HISTORY = new History(); private static JPanel historyPanel; private static JTable historyTable; @@ -79,12 +100,12 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static JButton historyButtonRemove, historyButtonClear, historyButtonRerip; // Queue - public static JButton optionQueue; + public static JToggleButton optionQueue; private static JPanel queuePanel; private static DefaultListModel queueListModel; // Configuration - private static JButton optionConfiguration; + private static JToggleButton optionConfiguration; private static JPanel configurationPanel; private static JButton configUpdateButton; private static JLabel configUpdateLabel; @@ -126,23 +147,39 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static Image mainIcon; + private static Function addUserInputUrlToQueueStatic; + private static Runnable ripNextAlbumStatic; + private static AbstractRipper ripper; - private void updateQueue(DefaultListModel model) { - if (model == null) - model = queueListModel; + private static final AtomicBoolean gracefulStop = new AtomicBoolean(false); // Allow active transfers to finish, then stop ripping. + private static final AtomicBoolean panicStop = new AtomicBoolean(false); // Immediately stop active transfers, then stop ripping. + private static final AtomicBoolean isRipperActive = new AtomicBoolean(false); - if (model.size() > 0) { - Utils.setConfigList("queue", model.elements()); - Utils.saveConfig(); - } + private final DebouncedRunnable debouncedSaveConfig = new DebouncedRunnable(Utils::saveConfig, 500); - MainWindow.optionQueue.setText(String.format("%s%s", Utils.getLocalizedString("queue"), - model.size() == 0 ? "" : "(" + model.size() + ")")); - } + public static final int TRANSFER_RATE_REFRESH_RATE = 200; + private static final TransferRate transferRate = new TransferRate(); + + private static final ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); + private Future rateRefresherFuture = null; + private final Runnable rateRefresher = () -> { + if (!isRipperActive.get()) { + if (rateRefresherFuture != null) { + rateRefresherFuture.cancel(true); + rateRefresherFuture = null; + } + transferRateValue.setText("0.00 B/s"); + return; + } + transferRateValue.setText(transferRate.formatHumanTransferRate()); + }; private void updateQueue() { - updateQueue(null); + Utils.setConfigList("queue", queueListModel.elements()); + debouncedSaveConfig.run(); + MainWindow.optionQueue.setText(String.format("%s%s", Utils.getLocalizedString("queue"), + queueListModel.isEmpty() ? "" : "(" + queueListModel.size() + ")")); } private static void addCheckboxListener(JCheckBox checkBox, String configString) { @@ -168,7 +205,7 @@ public MainWindow() throws IOException { mainFrame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); mainFrame.setLayout(new GridBagLayout()); - createUI(mainFrame.getContentPane()); + createUI((JPanel) mainFrame.getContentPane()); pack(); loadHistory(); @@ -198,6 +235,7 @@ public void run() { pack(); restoreWindowPosition(mainFrame); mainFrame.setVisible(true); + mainFrame.pack(); } private void shutdownCleanup() { @@ -234,6 +272,9 @@ private void error(String text) { } private void statusWithColor(String text, Color color) { + if (text == null || text.trim().isEmpty()) { + return; + } statusLabel.setForeground(color); statusLabel.setText(text); pack(); @@ -254,7 +295,11 @@ private boolean isCollapsed() { && !configurationPanel.isVisible()); } - private void createUI(Container pane) { + private void createUI(JPanel pane) { + + Insets buttonPadding = new Insets(2,2,2,2); + UIManager.getDefaults().put("Button.margin", buttonPadding); + // If creating the tray icon fails, ignore it. try { setupTrayIcon(); @@ -262,16 +307,7 @@ private void createUI(Container pane) { LOGGER.warn(e.getMessage()); } - EmptyBorder emptyBorder = new EmptyBorder(5, 5, 5, 5); - GridBagConstraints gbc = new GridBagConstraints(); - gbc.fill = GridBagConstraints.HORIZONTAL; - gbc.weightx = 1; - gbc.ipadx = 2; - gbc.gridx = 0; - gbc.weighty = 0; - gbc.ipady = 2; - gbc.gridy = 0; - gbc.anchor = GridBagConstraints.PAGE_START; + pane.setBorder(new EmptyBorder(5, 5, 5, 5)); try { UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName()); @@ -280,6 +316,8 @@ private void createUI(Container pane) { LOGGER.error("[!] Exception setting system theme:", e); } + Font monospaced = new Font(Font.MONOSPACED, Font.PLAIN, mainFrame.getContentPane().getFont().getSize()); + ripTextfield = new JTextField("", 20); ripTextfield.addMouseListener(new ContextMenuMouseListener(ripTextfield)); @@ -323,17 +361,20 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib */ ImageIcon ripIcon = new ImageIcon(mainIcon); - ripButton = new JButton("Rip", ripIcon); - stopButton = new JButton("Stop"); + ripButton = new JButton("Rip", ripIcon); + stopButton = new JButton("Stop"); stopButton.setEnabled(false); + panicButton = new JButton("Panic!"); + panicButton.setEnabled(false); try { Image stopIcon = ImageIO.read(getClass().getClassLoader().getResource("stop.png")); stopButton.setIcon(new ImageIcon(stopIcon)); } catch (Exception ignored) { } - JPanel ripPanel = new JPanel(new GridBagLayout()); - ripPanel.setBorder(emptyBorder); + GridBagConstraints gbc; + JPanel ripPanel = new JPanel(new GridBagLayout()); + gbc = new GridBagConstraints(); gbc.fill = GridBagConstraints.BOTH; gbc.weightx = 0; gbc.gridx = 0; @@ -348,32 +389,142 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib ripPanel.add(ripButton, gbc); gbc.gridx = 3; ripPanel.add(stopButton, gbc); + gbc.gridx = 4; + ripPanel.add(panicButton, gbc); gbc.weightx = 1; statusLabel = new JLabel(Utils.getLocalizedString("inactive")); + statusLabel.setFont(new Font(Font.DIALOG, Font.PLAIN, statusLabel.getFont().getSize())); statusLabel.setHorizontalAlignment(JLabel.CENTER); - openButton = new JButton(); + + JPanel statusDetailPanel = new JPanel(new GridBagLayout()); + + Border valueLabelBorder = BorderFactory.createEmptyBorder(0, 5, 0, 5); + + pendingValue.setFont(monospaced); + pendingValue.setHorizontalAlignment(JLabel.TRAILING); + pendingValue.setBorder(valueLabelBorder); + gbc = new GridBagConstraints(); + gbc.gridx = 0; + gbc.gridy = 0; + gbc.anchor = GridBagConstraints.EAST; + statusDetailPanel.add(pendingValue, gbc); + gbc = new GridBagConstraints(); + gbc.gridx = 1; + gbc.gridy = 0; + gbc.anchor = GridBagConstraints.WEST; + statusDetailPanel.add(pendingLabel, gbc); + activeValue.setFont(monospaced); + activeValue.setHorizontalAlignment(JLabel.TRAILING); + activeValue.setBorder(valueLabelBorder); + gbc = new GridBagConstraints(); + gbc.gridx = 0; + gbc.gridy = 1; + gbc.anchor = GridBagConstraints.EAST; + statusDetailPanel.add(activeValue, gbc); + gbc = new GridBagConstraints(); + gbc.gridx = 1; + gbc.gridy = 1; + gbc.anchor = GridBagConstraints.WEST; + statusDetailPanel.add(activeLabel, gbc); + completedValue.setFont(monospaced); + completedValue.setHorizontalAlignment(JLabel.TRAILING); + completedValue.setBorder(valueLabelBorder); + gbc = new GridBagConstraints(); + gbc.gridx = 3; + gbc.gridy = 0; + gbc.anchor = GridBagConstraints.EAST; + statusDetailPanel.add(completedValue, gbc); + gbc = new GridBagConstraints(); + gbc.gridx = 4; + gbc.gridy = 0; + gbc.anchor = GridBagConstraints.WEST; + statusDetailPanel.add(completedLabel, gbc); + erroredValue.setFont(monospaced); + erroredValue.setHorizontalAlignment(JLabel.TRAILING); + erroredValue.setBorder(valueLabelBorder); + gbc = new GridBagConstraints(); + gbc.gridx = 3; + gbc.gridy = 1; + gbc.anchor = GridBagConstraints.EAST; + statusDetailPanel.add(erroredValue, gbc); + gbc = new GridBagConstraints(); + gbc.gridx = 4; + gbc.gridy = 1; + gbc.anchor = GridBagConstraints.WEST; + statusDetailPanel.add(erroredLabel, gbc); + totalValue.setFont(monospaced); + totalValue.setHorizontalAlignment(JLabel.TRAILING); + totalValue.setBorder(valueLabelBorder); + gbc = new GridBagConstraints(); + gbc.gridx = 6; + gbc.gridy = 0; + gbc.anchor = GridBagConstraints.EAST; + statusDetailPanel.add(totalValue, gbc); + gbc = new GridBagConstraints(); + gbc.gridx = 7; + gbc.gridy = 0; + gbc.anchor = GridBagConstraints.WEST; + statusDetailPanel.add(totalLabel, gbc); + transferRateValue.setFont(monospaced); + transferRateValue.setHorizontalAlignment(JLabel.TRAILING); + transferRateValue.setBorder(valueLabelBorder); + gbc = new GridBagConstraints(); + gbc.gridx = 6; + gbc.gridy = 1; + gbc.anchor = GridBagConstraints.EAST; + statusDetailPanel.add(transferRateValue, gbc); + gbc = new GridBagConstraints(); + gbc.gridx = 7; + gbc.gridy = 1; + gbc.anchor = GridBagConstraints.WEST; + statusDetailPanel.add(transferRateLabel, gbc); + + final JPanel spacer1 = new JPanel(); + gbc = new GridBagConstraints(); + gbc.gridx = 2; + gbc.gridy = 0; + gbc.weightx = 1.0; + gbc.fill = GridBagConstraints.HORIZONTAL; + statusDetailPanel.add(spacer1, gbc); + final JPanel spacer2 = new JPanel(); + gbc = new GridBagConstraints(); + gbc.gridx = 5; + gbc.gridy = 0; + gbc.weightx = 1.0; + gbc.fill = GridBagConstraints.HORIZONTAL; + statusDetailPanel.add(spacer2, gbc); + openButton.setVisible(false); + + gbc = new GridBagConstraints(); JPanel statusPanel = new JPanel(new GridBagLayout()); - statusPanel.setBorder(emptyBorder); - gbc.gridx = 0; + gbc.weightx = 1.0; + gbc.gridy = 0; + gbc.fill = GridBagConstraints.HORIZONTAL; statusPanel.add(statusLabel, gbc); gbc.gridy = 1; + gbc.fill = GridBagConstraints.HORIZONTAL; + statusPanel.add(currentlyRippingProgress, gbc); + gbc.gridy = 2; + gbc.fill = GridBagConstraints.NONE; + statusPanel.add(statusDetailPanel, gbc); + gbc.gridy = 3; + gbc.fill = GridBagConstraints.HORIZONTAL; statusPanel.add(openButton, gbc); - gbc.gridy = 0; - - JPanel progressPanel = new JPanel(new GridBagLayout()); - progressPanel.setBorder(emptyBorder); - statusProgress = new JProgressBar(0, 100); - progressPanel.add(statusProgress, gbc); JPanel optionsPanel = new JPanel(new GridBagLayout()); - optionsPanel.setBorder(emptyBorder); - optionLog = new JButton(Utils.getLocalizedString("Log")); - optionHistory = new JButton(Utils.getLocalizedString("History")); - optionQueue = new JButton(Utils.getLocalizedString("queue")); - optionConfiguration = new JButton(Utils.getLocalizedString("Configuration")); + optionLog = new JToggleButton(Utils.getLocalizedString("Log")); + optionHistory = new JToggleButton(Utils.getLocalizedString("History")); + optionQueue = new JToggleButton(Utils.getLocalizedString("queue")); + optionConfiguration = new JToggleButton(Utils.getLocalizedString("Configuration")); + + panelButtonGroup.add(optionLog); + panelButtonGroup.add(optionHistory); + panelButtonGroup.add(optionQueue); + panelButtonGroup.add(optionConfiguration); + optionLog.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); optionHistory.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); optionQueue.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); @@ -391,6 +542,9 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib } catch (Exception e) { LOGGER.warn(e.getMessage()); } + + setTabButtonPreferredSizes(); + gbc.gridx = 0; optionsPanel.add(optionLog, gbc); gbc.gridx = 1; @@ -401,7 +555,6 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib optionsPanel.add(optionConfiguration, gbc); logPanel = new JPanel(new GridBagLayout()); - logPanel.setBorder(emptyBorder); logText = new JTextPane(); logText.setEditable(false); JScrollPane logTextScroll = new JScrollPane(logText); @@ -415,7 +568,6 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib gbc.weighty = 0; historyPanel = new JPanel(new GridBagLayout()); - historyPanel.setBorder(emptyBorder); historyPanel.setVisible(false); historyPanel.setPreferredSize(new Dimension(300, 250)); @@ -449,12 +601,12 @@ public int getColumnCount() { @Override public boolean isCellEditable(int row, int col) { - return (col == 0 || col == 4); + return (col == 0 || col == 5); } @Override public void setValueAt(Object value, int row, int col) { - if (col == 4) { + if (col == 5) { HISTORY.get(row).selected = (Boolean) value; historyTableModel.fireTableDataChanged(); } @@ -465,21 +617,12 @@ public void setValueAt(Object value, int row, int col) { historyTable.addMouseListener(new HistoryMenuMouseListener()); historyTable.setAutoCreateRowSorter(true); - for (int i = 0; i < historyTable.getColumnModel().getColumnCount(); i++) { - int width = 130; // Default - switch (i) { - case 0: // URL - width = 270; - break; - case 3: - width = 40; - break; - case 4: - width = 15; - break; - } - historyTable.getColumnModel().getColumn(i).setPreferredWidth(width); - } + historyTable.getColumnModel().getColumn(0).setPreferredWidth(270); // URL + //historyTable.getColumnModel().getColumn(1).setPreferredWidth(270); // Number + //historyTable.getColumnModel().getColumn(2).setPreferredWidth(130); // Date + //historyTable.getColumnModel().getColumn(3).setPreferredWidth(130); // Date + historyTable.getColumnModel().getColumn(4).setPreferredWidth(40); // Count + historyTable.getColumnModel().getColumn(5).setPreferredWidth(15); // Selected JScrollPane historyTableScrollPane = new JScrollPane(historyTable); historyButtonRemove = new JButton(Utils.getLocalizedString("remove")); @@ -497,7 +640,6 @@ public void setValueAt(Object value, int row, int col) { gbc.ipady = 0; JPanel historyButtonPanel = new JPanel(new GridBagLayout()); historyButtonPanel.setSize(new Dimension(300, 10)); - historyButtonPanel.setBorder(emptyBorder); gbc.gridx = 0; historyButtonPanel.add(historyButtonRemove, gbc); gbc.gridx = 1; @@ -511,20 +653,18 @@ public void setValueAt(Object value, int row, int col) { historyPanel.add(historyButtonPanel, gbc); queuePanel = new JPanel(new GridBagLayout()); - queuePanel.setBorder(emptyBorder); queuePanel.setVisible(false); queuePanel.setPreferredSize(new Dimension(300, 250)); queueListModel = new DefaultListModel<>(); JList queueList = new JList<>(queueListModel); queueList.setSelectionMode(ListSelectionModel.MULTIPLE_INTERVAL_SELECTION); - QueueMenuMouseListener queueMenuMouseListener = new QueueMenuMouseListener(d -> updateQueue(queueListModel)); + QueueMenuMouseListener queueMenuMouseListener = new QueueMenuMouseListener(); queueList.addMouseListener(queueMenuMouseListener); + queueList.addMouseMotionListener(queueMenuMouseListener); JScrollPane queueListScroll = new JScrollPane(queueList, JScrollPane.VERTICAL_SCROLLBAR_AS_NEEDED, JScrollPane.HORIZONTAL_SCROLLBAR_NEVER); - for (String item : Utils.getConfigList("queue")) { - queueListModel.addElement(item); - } + queueListModel.addAll(Utils.getConfigList("queue")); updateQueue(); gbc.gridx = 0; @@ -538,7 +678,6 @@ public void setValueAt(Object value, int row, int col) { gbc.ipady = 0; configurationPanel = new JPanel(new GridBagLayout()); - configurationPanel.setBorder(emptyBorder); configurationPanel.setVisible(false); // TODO Configuration components @@ -550,6 +689,21 @@ public void setValueAt(Object value, int row, int col) { configRetriesLabel = new JLabel(Utils.getLocalizedString("retry.download.count"), JLabel.RIGHT); configRetrySleepLabel = new JLabel(Utils.getLocalizedString("retry.sleep.mill"), JLabel.RIGHT); configThreadsText = configField("threads.size", 3); + configThreadsText.addActionListener(e -> { + Document document = configThreadsText.getDocument(); + LOGGER.info("Updating thread pool size"); + if (ripper != null && document != null) { + String text = configThreadsText.getText().trim(); + try { + int threads = Integer.parseInt(text); + if (threads >= 0) { + ripper.setThreadPoolSize(threads); + } + } catch (NumberFormatException ex) { + // ignore invalid input + } + } + }); configTimeoutText = configField("download.timeout", 60000); configRetriesText = configField("download.retries", 3); configRetrySleepText = configField("download.retry.sleep", 5000); @@ -620,29 +774,43 @@ public void setValueAt(Object value, int row, int col) { emptyPanel.setPreferredSize(new Dimension(0, 0)); emptyPanel.setSize(0, 0); - gbc.anchor = GridBagConstraints.PAGE_START; + optionLog.putClientProperty(RIPME_PANEL, logPanel); + optionHistory.putClientProperty(RIPME_PANEL, historyPanel); + optionQueue.putClientProperty(RIPME_PANEL, queuePanel); + optionConfiguration.putClientProperty(RIPME_PANEL, configurationPanel); + gbc.gridy = 0; + gbc.gridx = 0; pane.add(ripPanel, gbc); gbc.gridy = 1; pane.add(statusPanel, gbc); gbc.gridy = 2; - pane.add(progressPanel, gbc); - gbc.gridy = 3; pane.add(optionsPanel, gbc); gbc.weighty = 1; gbc.fill = GridBagConstraints.BOTH; - gbc.gridy = 4; + gbc.gridy = 3; pane.add(logPanel, gbc); - gbc.gridy = 5; pane.add(historyPanel, gbc); - gbc.gridy = 5; pane.add(queuePanel, gbc); - gbc.gridy = 5; pane.add(configurationPanel, gbc); - gbc.gridy = 5; pane.add(emptyPanel, gbc); - gbc.weighty = 0; - gbc.fill = GridBagConstraints.HORIZONTAL; + } + + private static void setTabButtonPreferredSizes() { + // Recalculate preferred size each time by using getUI() + + // Prevent button sizes/positions from shifting when text bolds/unbolds + optionLog.setPreferredSize(optionLog.getUI().getPreferredSize(optionLog)); + optionHistory.setPreferredSize(optionHistory.getUI().getPreferredSize(optionHistory)); + + // Set preferred size with space for queue size string + optionQueue.setText(Utils.getLocalizedString("queue") + " (8888)"); + optionQueue.setPreferredSize(optionQueue.getUI().getPreferredSize(optionQueue)); + // Restore original text + optionQueue.setText(Utils.getLocalizedString("queue")); + // updateQueue() is called below, which initializes the real queue button text + + optionConfiguration.setPreferredSize(optionConfiguration.getUI().getPreferredSize(optionConfiguration)); } private JTextField configField(String key, int defaultValue) { @@ -678,8 +846,8 @@ private void checkAndUpdate() { return field; } - private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYValue, JLabel thing1ToAdd, - JButton thing2ToAdd) { + private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYValue, JComponent thing1ToAdd, + JComponent thing2ToAdd) { gbc.gridy = gbcYValue; gbc.gridx = 0; configurationPanel.add(thing1ToAdd, gbc); @@ -687,51 +855,6 @@ private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYV configurationPanel.add(thing2ToAdd, gbc); } - private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYValue, JLabel thing1ToAdd, - JTextField thing2ToAdd) { - gbc.gridy = gbcYValue; - gbc.gridx = 0; - configurationPanel.add(thing1ToAdd, gbc); - gbc.gridx = 1; - configurationPanel.add(thing2ToAdd, gbc); - } - - private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYValue, JCheckBox thing1ToAdd, - JCheckBox thing2ToAdd) { - gbc.gridy = gbcYValue; - gbc.gridx = 0; - configurationPanel.add(thing1ToAdd, gbc); - gbc.gridx = 1; - configurationPanel.add(thing2ToAdd, gbc); - } - - @SuppressWarnings("rawtypes") - private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYValue, JCheckBox thing1ToAdd, - JComboBox thing2ToAdd) { - gbc.gridy = gbcYValue; - gbc.gridx = 0; - configurationPanel.add(thing1ToAdd, gbc); - gbc.gridx = 1; - configurationPanel.add(thing2ToAdd, gbc); - } - - @SuppressWarnings("rawtypes") - private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYValue, JComboBox thing1ToAdd, - JButton thing2ToAdd) { - gbc.gridy = gbcYValue; - gbc.gridx = 0; - configurationPanel.add(thing1ToAdd, gbc); - gbc.gridx = 1; - configurationPanel.add(thing2ToAdd, gbc); - } - - @SuppressWarnings({ "unused", "rawtypes" }) - private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYValue, JComboBox thing1ToAdd) { - gbc.gridy = gbcYValue; - gbc.gridx = 0; - configurationPanel.add(thing1ToAdd, gbc); - } - private void changeLocale() { statusLabel.setText(Utils.getLocalizedString("inactive")); configUpdateButton.setText(Utils.getLocalizedString("check.for.updates")); @@ -759,9 +882,23 @@ private void changeLocale() { optionHistory.setText(Utils.getLocalizedString("History")); optionQueue.setText(Utils.getLocalizedString("queue")); optionConfiguration.setText(Utils.getLocalizedString("Configuration")); + ripButton.setText(Utils.getLocalizedString("Rip")); + stopButton.setText(Utils.getLocalizedString("Stop")); + panicButton.setText(Utils.getLocalizedString("Panic")); + + pendingLabel.setText(Utils.getLocalizedString("Pending")); + activeLabel.setText(Utils.getLocalizedString("Active")); + completedLabel.setText(Utils.getLocalizedString("Completed")); + erroredLabel.setText(Utils.getLocalizedString("Errored")); + totalLabel.setText(Utils.getLocalizedString("Total")); + transferRateLabel.setText(Utils.getLocalizedString("Speed")); + + setTabButtonPreferredSizes(); // Preferred size may change with different width labels } private void setupHandlers() { + addUserInputUrlToQueueStatic = this::addUserInputUrlToQueue; + ripNextAlbumStatic = this::ripNextAlbum; ripButton.addActionListener(new RipButtonHandler(this)); ripTextfield.addActionListener(new RipButtonHandler(this)); ripTextfield.getDocument().addDocumentListener(new DocumentListener() { @@ -801,84 +938,56 @@ private void update() { stopButton.addActionListener(event -> { if (ripper != null) { ripper.stop(); - isRipping = false; + gracefulStop.set(true); + queueListModel.add(0, ripper.getURL().toString()); stopButton.setEnabled(false); - statusProgress.setValue(0); - statusProgress.setVisible(false); pack(); - statusProgress.setValue(0); - status(Utils.getLocalizedString("download.interrupted")); - appendLog("Download interrupted", Color.RED); + status(Utils.getLocalizedString("rip.gracefully.stopping")); + appendLog(Utils.getLocalizedString("download.interrupted"), Color.RED); } }); - optionLog.addActionListener(event -> { - logPanel.setVisible(!logPanel.isVisible()); - emptyPanel.setVisible(!logPanel.isVisible()); - historyPanel.setVisible(false); - queuePanel.setVisible(false); - configurationPanel.setVisible(false); - if (logPanel.isVisible()) { - optionLog.setFont(optionLog.getFont().deriveFont(Font.BOLD)); - } else { - optionLog.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); + panicButton.addActionListener(event -> { + if (ripper != null) { + ripper.stop(); + ripper.panic(); + panicStop.set(true); + queueListModel.add(0, ripper.getURL().toString()); + stopButton.setEnabled(false); + panicButton.setEnabled(false); + currentlyRippingProgress.setValue(0); + currentlyRippingProgress.setText(""); + pack(); + status(Utils.getLocalizedString("rip.interrupted")); + appendLog(Utils.getLocalizedString("download.interrupted"), Color.RED); } - optionHistory.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); - optionQueue.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); - optionConfiguration.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); - pack(); }); - optionHistory.addActionListener(event -> { - logPanel.setVisible(false); - historyPanel.setVisible(!historyPanel.isVisible()); - emptyPanel.setVisible(!historyPanel.isVisible()); - queuePanel.setVisible(false); - configurationPanel.setVisible(false); - optionLog.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); - if (historyPanel.isVisible()) { - optionHistory.setFont(optionLog.getFont().deriveFont(Font.BOLD)); - } else { - optionHistory.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); + openButton.addActionListener(event -> { + try { + Utils.open(new File(event.getActionCommand())); + } catch (Exception e) { + LOGGER.error(e); } - optionQueue.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); - optionConfiguration.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); - pack(); }); - optionQueue.addActionListener(event -> { - logPanel.setVisible(false); - historyPanel.setVisible(false); - queuePanel.setVisible(!queuePanel.isVisible()); - emptyPanel.setVisible(!queuePanel.isVisible()); - configurationPanel.setVisible(false); - optionLog.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); - optionHistory.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); - if (queuePanel.isVisible()) { - optionQueue.setFont(optionLog.getFont().deriveFont(Font.BOLD)); - } else { - optionQueue.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); + ActionListener panelSelectListener = event -> { + JToggleButton source = (JToggleButton) event.getSource(); + Enumeration buttons = panelButtonGroup.getElements(); + while (buttons.hasMoreElements()) { + AbstractButton button = buttons.nextElement(); + JPanel tabPanel = (JPanel) button.getClientProperty(RIPME_PANEL); + boolean visible = button == source && source.isSelected(); + tabPanel.setVisible(visible); } - optionConfiguration.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); + emptyPanel.setVisible(!source.isSelected()); pack(); - }); + }; - optionConfiguration.addActionListener(event -> { - logPanel.setVisible(false); - historyPanel.setVisible(false); - queuePanel.setVisible(false); - configurationPanel.setVisible(!configurationPanel.isVisible()); - emptyPanel.setVisible(!configurationPanel.isVisible()); - optionLog.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); - optionHistory.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); - optionQueue.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); - if (configurationPanel.isVisible()) { - optionConfiguration.setFont(optionLog.getFont().deriveFont(Font.BOLD)); - } else { - optionConfiguration.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); - } - pack(); - }); + optionLog.addActionListener(panelSelectListener); + optionHistory.addActionListener(panelSelectListener); + optionQueue.addActionListener(panelSelectListener); + optionConfiguration.addActionListener(panelSelectListener); historyButtonRemove.addActionListener(event -> { int[] indices = historyTable.getSelectedRows(); @@ -979,8 +1088,7 @@ public void mouseClicked(MouseEvent e) { Path file; try { file = Utils.getWorkingDirectory(); - Desktop desktop = Desktop.getDesktop(); - desktop.open(file.toFile()); + Utils.open(file.toFile()); } catch (IOException ex) { LOGGER.warn(ex.getMessage()); } @@ -1060,41 +1168,30 @@ public void mouseClicked(MouseEvent e) { @Override public void intervalAdded(ListDataEvent arg0) { updateQueue(); - - if (!isRipping) { - ripNextAlbum(); - } + ripNextAlbum(); } @Override public void contentsChanged(ListDataEvent arg0) { + updateQueue(); } @Override public void intervalRemoved(ListDataEvent arg0) { + updateQueue(); } }); } private void setLogLevel(String level) { - // default level is error, set in case something else is given. - Level newLevel = Level.ERROR; level = level.substring(level.lastIndexOf(' ') + 1); - switch (level) { - case "Debug": - newLevel = Level.DEBUG; - break; - case "Info": - newLevel = Level.INFO; - break; - case "Warn": - newLevel = Level.WARN; - } - LoggerContext ctx = (LoggerContext) LogManager.getContext(false); - Configuration config = ctx.getConfiguration(); - LoggerConfig loggerConfig = config.getLoggerConfig(LogManager.ROOT_LOGGER_NAME); - loggerConfig.setLevel(newLevel); - ctx.updateLoggers(); // This causes all Loggers to refetch information from their LoggerConfig. + Level newLevel = switch (level) { + case "Debug" -> Level.DEBUG; + case "Info" -> Level.INFO; + case "Warn" -> Level.WARN; + default -> Level.ERROR; + }; + Utils.configureLogger(newLevel); } private void setupTrayIcon() { @@ -1174,7 +1271,7 @@ public void windowIconified(WindowEvent e) { JOptionPane.YES_NO_OPTION, JOptionPane.PLAIN_MESSAGE, new ImageIcon(mainIcon)); if (response == JOptionPane.YES_OPTION) { try { - Desktop.getDesktop().browse(URI.create("http://github.com/ripmeapp/ripme")); + Utils.browse(URI.create("https://github.com/ripmeapp/ripme")); } catch (IOException e) { LOGGER.error("Exception while opening project home page", e); } @@ -1247,7 +1344,11 @@ private void appendLog(final String text, final Color color) { StyledDocument sd = logText.getStyledDocument(); try { synchronized (this) { + if (logLineLengths.size() > MAX_LOG_PANE_LINES) { + sd.remove(0, logLineLengths.remove()); + } sd.insertString(sd.getLength(), text + "\n", sas); + logLineLengths.add(text.length() + 1); } } catch (BadLocationException e) { LOGGER.warn(e.getMessage()); @@ -1274,6 +1375,7 @@ private void loadHistory() throws IOException { try { LOGGER.info(Utils.getLocalizedString("loading.history.from") + " " + historyFile.getCanonicalPath()); HISTORY.fromFile(historyFile.getCanonicalPath()); + LOGGER.info("Finished loading history"); } catch (IOException e) { LOGGER.error("Failed to load history from file " + historyFile, e); JOptionPane.showMessageDialog(null, @@ -1301,6 +1403,52 @@ private void loadHistory() throws IOException { }); } } + if (!HISTORY.isEmpty()) { + // Fix "WARNING: row index is bigger than sorter's row count. Most likely this is a wrong sorter usage" + historyTableModel.fireTableDataChanged(); + } + + // Calculate preferred column widths + int autoResizeMode = historyTable.getAutoResizeMode(); + historyTable.setAutoResizeMode(JTable.AUTO_RESIZE_OFF); + int lastRow = historyTable.getRowCount() - 1; + int nColumn = 1; + TableCellRenderer renderer; + renderer = historyTable.getCellRenderer(lastRow, nColumn); + Component comp = historyTable.prepareRenderer(renderer, lastRow, nColumn); + int width = Math.min(Math.max(15, comp.getMinimumSize().width + 15), 130); + historyTable.getColumnModel().getColumn(nColumn).setPreferredWidth(width); + historyTable.getColumnModel().getColumn(nColumn).setMaxWidth(130); + + renderer = historyTable.getDefaultRenderer(String.class); + int cWidth; + int column; + column = 2; // date + cWidth = renderer.getTableCellRendererComponent( + historyTable, "8888/88/88", false, false, 0, column) + .getPreferredSize().width + 15; + historyTable.getColumnModel().getColumn(column).setPreferredWidth(cWidth); + historyTable.getColumnModel().getColumn(column).setMaxWidth(cWidth); + column = 3; // date + cWidth = renderer.getTableCellRendererComponent( + historyTable, "8888/88/88", false, false, 0, column) + .getPreferredSize().width + 15; + historyTable.getColumnModel().getColumn(column).setPreferredWidth(cWidth); + historyTable.getColumnModel().getColumn(column).setMaxWidth(cWidth); + column = 4; // count + cWidth = renderer.getTableCellRendererComponent( + historyTable, "88888", false, false, 0, column) + .getPreferredSize().width + 15; + historyTable.getColumnModel().getColumn(column).setMaxWidth(cWidth); + + renderer = historyTable.getDefaultRenderer(Boolean.class); + column = 5; // selected + cWidth = renderer.getTableCellRendererComponent( + historyTable, true, false, false, 0, column) + .getPreferredSize().width; + historyTable.getColumnModel().getColumn(column).setMaxWidth(cWidth); + + historyTable.setAutoResizeMode(autoResizeMode); } private void saveHistory() { @@ -1319,40 +1467,73 @@ private void saveHistory() { } private void ripNextAlbum() { - isRipping = true; + LOGGER.debug("ripNextAlbum called"); + if (isRipperActive.getAndSet(true)) { + // Already ripping + LOGGER.debug("already ripping"); + return; + } // Save current state of queue to configuration. Utils.setConfigList("queue", queueListModel.elements()); + boolean wasGracefulStop = gracefulStop.getAndSet(false); + boolean wasPanicStop = gracefulStop.getAndSet(false); + if (wasGracefulStop || wasPanicStop) { + // Stop requested + LOGGER.debug("wasGracefulStop or wasPanicStop"); + ripFinishCleanup(); + return; + } + if (queueListModel.isEmpty()) { // End of queue - isRipping = false; + ripFinishCleanup(); return; } + if (rateRefresherFuture == null || rateRefresherFuture.isDone()) { + rateRefresherFuture = executor.scheduleAtFixedRate(rateRefresher, 0, TRANSFER_RATE_REFRESH_RATE, TimeUnit.MILLISECONDS); + } + String nextAlbum = (String) queueListModel.remove(0); updateQueue(); + LOGGER.debug("calling ripAlbum(\"{}\")", nextAlbum); Thread t = ripAlbum(nextAlbum); if (t == null) { + LOGGER.debug("ripAlbum() returned null"); try { Thread.sleep(500); } catch (InterruptedException ie) { LOGGER.error(Utils.getLocalizedString("interrupted.while.waiting.to.rip.next.album"), ie); } + isRipperActive.set(false); ripNextAlbum(); } else { + LOGGER.debug("Starting new ripper thread"); t.start(); } } + private void ripFinishCleanup() { + stopButton.setEnabled(false); + panicButton.setEnabled(false); + isRipperActive.set(false); + currentlyRippingProgress.setValue(0); + currentlyRippingProgress.setText(""); + } + private Thread ripAlbum(String urlString) { - if (!logPanel.isVisible()) { + boolean isAnyPanelOpen = panelButtonGroup.getSelection() != null; + if (!isAnyPanelOpen) { optionLog.doClick(); } urlString = urlString.trim(); + LOGGER.info("Attempting to start rip for album {}", urlString); + appendLog(Utils.getLocalizedString("attempting.to.start.rip.for.album.0", urlString), Color.GREEN); if (urlString.toLowerCase().startsWith("gonewild:")) { urlString = "http://gonewild.com/user/" + urlString.substring(urlString.indexOf(':') + 1); } @@ -1368,12 +1549,22 @@ private Thread ripAlbum(String urlString) { return null; } stopButton.setEnabled(true); - statusProgress.setValue(100); + panicButton.setEnabled(true); + currentlyRippingProgress.setValue(0); + currentlyRippingProgress.setText(urlString); openButton.setVisible(false); statusLabel.setVisible(true); + + pendingValue.setText("0"); + activeValue.setText("0"); + completedValue.setText("0"); + totalValue.setText("0"); + erroredValue.setText("0"); + pack(); boolean failed = false; try { + LOGGER.debug("Creating ripper for url {}", url); ripper = AbstractRipper.getRipper(url); ripper.setup(); } catch (Exception e) { @@ -1405,12 +1596,12 @@ private Thread ripAlbum(String urlString) { } } stopButton.setEnabled(false); - statusProgress.setValue(0); + currentlyRippingProgress.setValue(0); pack(); return null; } - private boolean canRip(String urlString) { + private static boolean canRip(String urlString) { try { String urlText = urlString.trim(); if (urlText.equals("")) { @@ -1440,6 +1631,47 @@ public static DefaultListModel getQueueListModel() { return queueListModel; } + /** + * @param url User input that might be a URL + * @return true if the URL is in the queue + */ + private boolean addUserInputUrlToQueue(String url) { + boolean urlInQueue = false; + boolean url_not_empty = !url.equals(""); + if (!queueListModel.contains(url) && url_not_empty) { + // Check if we're ripping a range of urls + if (url.contains("{")) { + // Make sure the user hasn't forgotten the closing } + if (url.contains("}")) { + String rangeToParse = url.substring(url.indexOf("{") + 1, url.indexOf("}")); + int rangeStart = Integer.parseInt(rangeToParse.split("-")[0]); + int rangeEnd = Integer.parseInt(rangeToParse.split("-")[1]); + for (int i = rangeStart; i < rangeEnd + 1; i++) { + String realURL = url.replaceAll("\\{\\S*\\}", Integer.toString(i)); + if (canRip(realURL)) { + queueListModel.addElement(realURL); + urlInQueue = true; + } else { + displayAndLogError("Can't find ripper for " + realURL, Color.RED); + } + } + } + } else { + if (canRip(url)) { + queueListModel.addElement(url); + urlInQueue = true; + } else { + displayAndLogError("Can't find ripper for " + url, Color.RED); + } + } + } else if (url_not_empty) { + displayAndLogError("This URL is already in queue: " + url, Color.RED); + statusWithColor("This URL is already in queue: " + url, Color.ORANGE); + urlInQueue = true; + } + return urlInQueue; + } + static class RipButtonHandler implements ActionListener { private MainWindow mainWindow; @@ -1449,37 +1681,11 @@ public RipButtonHandler(MainWindow mainWindow) { public void actionPerformed(ActionEvent event) { String url = ripTextfield.getText(); - boolean url_not_empty = !url.equals(""); - if (!queueListModel.contains(url) && url_not_empty) { - // Check if we're ripping a range of urls - if (url.contains("{")) { - // Make sure the user hasn't forgotten the closing } - if (url.contains("}")) { - String rangeToParse = url.substring(url.indexOf("{") + 1, url.indexOf("}")); - int rangeStart = Integer.parseInt(rangeToParse.split("-")[0]); - int rangeEnd = Integer.parseInt(rangeToParse.split("-")[1]); - for (int i = rangeStart; i < rangeEnd + 1; i++) { - String realURL = url.replaceAll("\\{\\S*\\}", Integer.toString(i)); - if (mainWindow.canRip(realURL)) { - queueListModel.addElement(realURL); - ripTextfield.setText(""); - } else { - mainWindow.displayAndLogError("Can't find ripper for " + realURL, Color.RED); - } - } - } - } else { - queueListModel.addElement(url); - ripTextfield.setText(""); - } - } else if (url_not_empty) { - mainWindow.displayAndLogError("This URL is already in queue: " + url, Color.RED); - mainWindow.statusWithColor("This URL is already in queue: " + url, Color.ORANGE); + boolean urlInQueue = mainWindow.addUserInputUrlToQueue(url); + if (urlInQueue) { ripTextfield.setText(""); } - else if(!mainWindow.isRipping){ - mainWindow.ripNextAlbum(); - } + mainWindow.ripNextAlbum(); } } @@ -1497,19 +1703,54 @@ public void run() { } } + private long lastStatusUpdate = 0; + private synchronized void handleEvent(StatusEvent evt) { - if (ripper.isStopped()) { - return; + RipStatusMessage msg = evt.msg; + RipStatusMessage.STATUS status = msg.getStatus(); + + // CHUNK_BYTES is noisy, so handle it before any other computation + if (status == RipStatusMessage.STATUS.CHUNK_BYTES) { + transferRate.addChunk((Long) msg.getObject()); + transferRateValue.setText(transferRate.formatHumanTransferRate()); + + // Quick hack: ripper.getActiveCount() and dependent values are approximate and the value can be outdated (too large) when it is called when the ripper notifies COMPLETE, + // so allow the status info to update too, but throttle it a little. + long now = System.currentTimeMillis(); + boolean allowStatusUpdate = now > lastStatusUpdate + 200; + if (!allowStatusUpdate) { + return; + } + lastStatusUpdate = now; } - RipStatusMessage msg = evt.msg; + if (evt.ripper.useByteProgessBar()) { + long bytesTotal = evt.ripper.getBytesTotal(); + long bytesCompleted = evt.ripper.getBytesCompleted(); + pendingValue.setText(Utils.bytesToHumanReadable(bytesTotal - bytesCompleted)); + completedValue.setText(Utils.bytesToHumanReadable(bytesCompleted)); + totalValue.setText(Utils.bytesToHumanReadable(bytesTotal)); + currentlyRippingProgress.setValue(evt.ripper.getCompletionPercentage()); + } else { + int pendingCount = evt.ripper.getPendingCount(); + int activeCount = evt.ripper.getActiveCount(); // included in pendingCount + int completedCount = evt.ripper.getCompletedCount(); + int erroredCount = evt.ripper.getErroredCount(); + int totalCount = evt.ripper.getTotalCount(); + pendingValue.setText(String.valueOf(Math.max(0, pendingCount - activeCount))); // exclude active + activeValue.setText(String.valueOf(activeCount)); + completedValue.setText(String.valueOf(completedCount)); + erroredValue.setText(String.valueOf(erroredCount)); + totalValue.setText(String.valueOf(totalCount)); + currentlyRippingProgress.setValue(evt.ripper.getCompletionPercentage()); + } - int completedPercent = evt.ripper.getCompletionPercentage(); - statusProgress.setValue(completedPercent); - statusProgress.setVisible(true); - status(evt.ripper.getStatusText()); + // Quick hack finish: + if (status == RipStatusMessage.STATUS.CHUNK_BYTES) { + return; + } - switch (msg.getStatus()) { + switch (status) { case LOADING_RESOURCE: case DOWNLOAD_STARTED: if (LOGGER.isEnabled(Level.INFO)) { @@ -1547,12 +1788,11 @@ private synchronized void handleEvent(StatusEvent evt) { if (LOGGER.isEnabled(Level.ERROR)) { appendLog((String) msg.getObject(), Color.RED); } - stopButton.setEnabled(false); - statusProgress.setValue(0); - statusProgress.setVisible(false); openButton.setVisible(false); + ripFinishCleanup(); pack(); statusWithColor("Error: " + msg.getObject(), Color.RED); + ripNextAlbum(); break; case RIP_COMPLETE: @@ -1580,9 +1820,7 @@ private synchronized void handleEvent(StatusEvent evt) { Utils.playSound("camera.wav"); } saveHistory(); - stopButton.setEnabled(false); - statusProgress.setValue(0); - statusProgress.setVisible(false); + Utils.saveConfig(); openButton.setVisible(true); Path f = rsc.dir; String prettyFile = Utils.shortenPath(f); @@ -1631,14 +1869,10 @@ private synchronized void handleEvent(StatusEvent evt) { } } appendLog("Rip complete, saved to " + f, Color.GREEN); + LOGGER.info("Rip complete: {}", url); + status(Utils.getLocalizedString("inactive")); openButton.setActionCommand(f.toString()); - openButton.addActionListener(event -> { - try { - Desktop.getDesktop().open(new File(event.getActionCommand())); - } catch (Exception e) { - LOGGER.error(e); - } - }); + ripFinishCleanup(); pack(); ripNextAlbum(); break; @@ -1652,10 +1886,8 @@ private synchronized void handleEvent(StatusEvent evt) { if (LOGGER.isEnabled(Level.ERROR)) { appendLog((String) msg.getObject(), Color.RED); } - stopButton.setEnabled(false); - statusProgress.setValue(0); - statusProgress.setVisible(false); openButton.setVisible(false); + ripFinishCleanup(); pack(); statusWithColor("Error: " + msg.getObject(), Color.RED); break; @@ -1668,8 +1900,10 @@ public void update(AbstractRipper ripper, RipStatusMessage message) { } public static void ripAlbumStatic(String url) { - ripTextfield.setText(url.trim()); - ripButton.doClick(); + boolean urlInQueue = addUserInputUrlToQueueStatic.apply(url.trim()); + if (urlInQueue) { + ripNextAlbumStatic.run(); + } } private static boolean hasWindowPositionBug() { diff --git a/src/main/java/com/rarchives/ripme/ui/MinimumWidthLabel.java b/src/main/java/com/rarchives/ripme/ui/MinimumWidthLabel.java new file mode 100644 index 000000000..0cae84c07 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ui/MinimumWidthLabel.java @@ -0,0 +1,45 @@ +package com.rarchives.ripme.ui; + +import javax.swing.JLabel; +import java.awt.Dimension; +import java.util.Objects; + +/** + * GridBagLayout does not respect minimum size, only preferred size. + * In order to set a minimum width, we need to override getPreferredSize. + */ +public class MinimumWidthLabel extends JLabel { + private String minimumWidthText; + private String currentText; + + public MinimumWidthLabel(String minimumWidthText, String defaultText) { + this.minimumWidthText = minimumWidthText; + setText(defaultText); + } + + @Override + public Dimension getPreferredSize() { + String text = getText(); + Dimension preferredSize = super.getPreferredSize(); + setText(minimumWidthText); + int minimumWidth = super.getPreferredSize().width; + // Hopefully GridBagLayout correctly handles maximum size and we don't need to clamp + preferredSize.width = Math.max(minimumWidth, preferredSize.width); + setText(text); + return preferredSize; + } + + public void setMinimumWidthText(String minimumWidthText) { + this.minimumWidthText = minimumWidthText; + } + + @Override + public void setText(String text) { + // Cache the last text to save cycles, + // because this may be called with the same value very often + if (!Objects.equals(currentText, text)) { + currentText = text; + super.setText(text); + } + } +} diff --git a/src/main/java/com/rarchives/ripme/ui/ProgressTextField.java b/src/main/java/com/rarchives/ripme/ui/ProgressTextField.java new file mode 100644 index 000000000..74b23bc8e --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ui/ProgressTextField.java @@ -0,0 +1,58 @@ +package com.rarchives.ripme.ui; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import javax.swing.*; +import java.awt.BorderLayout; +import java.awt.Color; +import java.awt.Font; + +public class ProgressTextField extends JProgressBar { + private static final Logger logger = LogManager.getLogger(ProgressTextField.class); + private final JTextField textField = new JTextField(); + private final JLabel valueLabel = new JLabel(); + + public ProgressTextField() { + super(0, 100); + setLayout(new BorderLayout()); + + // Paint an empty string to reserve height for the text field + super.setStringPainted(true); + progressString = ""; // Directly set here so setString can be overridden + + textField.setOpaque(false); + textField.setEditable(false); + textField.setBorder(BorderFactory.createEmptyBorder(0, 5, 0, 0)); + add(textField, BorderLayout.CENTER); + + valueLabel.setFont(new Font(Font.MONOSPACED, Font.PLAIN, valueLabel.getFont().getSize())); + valueLabel.setBorder(BorderFactory.createCompoundBorder( + BorderFactory.createMatteBorder(0, 1, 0, 0, Color.GRAY), + BorderFactory.createEmptyBorder(0, 5, 0, 5) + )); + setValue(0); + + add(valueLabel, BorderLayout.EAST); + } + + @Override + public void setString(String s) { + if (s != null) { + // super() calls setString(null) during construction; not a problem + logger.warn("bug: progress bar should use setText, not setString"); + } + } + + @Override + public void setValue(int n) { + super.setValue(n); + int minimum = getMinimum(); + // note: integer division + valueLabel.setText(100 * (n - minimum) / (getMaximum() - minimum) + "%"); + } + + public void setText(String t) { + textField.setText(t); + } +} diff --git a/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java b/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java index 0be4b46f8..de794730c 100644 --- a/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java +++ b/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java @@ -1,10 +1,9 @@ package com.rarchives.ripme.ui; import java.awt.event.ActionEvent; -import java.awt.event.InputEvent; import java.awt.event.MouseAdapter; import java.awt.event.MouseEvent; -import java.util.function.Consumer; +import java.util.List; import javax.swing.AbstractAction; import javax.swing.Action; @@ -12,6 +11,7 @@ import javax.swing.JList; import javax.swing.JOptionPane; import javax.swing.JPopupMenu; +import javax.swing.SwingUtilities; import com.rarchives.ripme.utils.Utils; @@ -19,26 +19,28 @@ class QueueMenuMouseListener extends MouseAdapter { private JPopupMenu popup = new JPopupMenu(); private JList queueList; private DefaultListModel queueListModel; - private Consumer> updateQueue; + private boolean mouseDragging = false; + private int dragSourceIndex; - public QueueMenuMouseListener(Consumer> updateQueue) { - this.updateQueue = updateQueue; + public QueueMenuMouseListener() { updateUI(); } - @SuppressWarnings("serial") public void updateUI() { popup.removeAll(); Action removeSelected = new AbstractAction(Utils.getLocalizedString("queue.remove.selected")) { @Override public void actionPerformed(ActionEvent ae) { - Object o = queueList.getSelectedValue(); - while (o != null) { - queueListModel.removeElement(o); - o = queueList.getSelectedValue(); + if (JOptionPane.showConfirmDialog(null, Utils.getLocalizedString("queue.remove.selected.validation"), "RipMe", + JOptionPane.YES_NO_OPTION) == JOptionPane.YES_OPTION) { + Object o = queueList.getSelectedValue(); + while (o != null) { + queueListModel.removeElement(o); + o = queueList.getSelectedValue(); + } + updateUI(); } - updateUI(); } }; popup.add(removeSelected); @@ -46,7 +48,7 @@ public void actionPerformed(ActionEvent ae) { Action clearQueue = new AbstractAction(Utils.getLocalizedString("queue.remove.all")) { @Override public void actionPerformed(ActionEvent ae) { - if (JOptionPane.showConfirmDialog(null, Utils.getLocalizedString("queue.validation"), "RipMe", + if (JOptionPane.showConfirmDialog(null, Utils.getLocalizedString("queue.remove.all.validation"), "RipMe", JOptionPane.YES_NO_OPTION) == JOptionPane.YES_OPTION) { queueListModel.removeAllElements(); updateUI(); @@ -55,22 +57,90 @@ public void actionPerformed(ActionEvent ae) { }; popup.add(clearQueue); - updateQueue.accept(queueListModel); + Action moveSelectedToTop = new AbstractAction(Utils.getLocalizedString("queue.move.selected.to.top")) { + @Override + public void actionPerformed(ActionEvent ae) { + List selectedElements = queueList.getSelectedValuesList(); + for (Object selectedElement : selectedElements) { + queueListModel.removeElement(selectedElement); + } + queueListModel.addAll(0, selectedElements); + queueList.setSelectionInterval(0, selectedElements.size() - 1); + updateUI(); + } + }; + popup.add(moveSelectedToTop); + + Action moveSelectedToBottom = new AbstractAction(Utils.getLocalizedString("queue.move.selected.to.bottom")) { + @Override + public void actionPerformed(ActionEvent ae) { + List selectedElements = queueList.getSelectedValuesList(); + for (Object selectedElement : selectedElements) { + queueListModel.removeElement(selectedElement); + } + queueListModel.addAll(selectedElements); + queueList.setSelectionInterval(queueListModel.size() - selectedElements.size(), queueListModel.size() - 1); + updateUI(); + } + }; + popup.add(moveSelectedToBottom); } @Override public void mousePressed(MouseEvent e) { checkPopupTrigger(e); + handleDragStart(e); } @Override public void mouseReleased(MouseEvent e) { checkPopupTrigger(e); + handleDragEnd(e); + } + + @SuppressWarnings("unchecked") + public void handleDragStart(MouseEvent e) { + if (!(e.getSource() instanceof JList)) { + return; + } + if (SwingUtilities.isLeftMouseButton(e)) { + queueList = (JList) e.getSource(); + queueListModel = (DefaultListModel) queueList.getModel(); + + dragSourceIndex = queueList.getSelectedIndex(); + mouseDragging = true; + } + } + + public void handleDragEnd(MouseEvent e) { + mouseDragging = false; + } + + @Override + @SuppressWarnings("unchecked") + public void mouseDragged(MouseEvent e) { + if (!(e.getSource() instanceof JList)) { + return; + } + if (mouseDragging) { + queueList = (JList) e.getSource(); + queueListModel = (DefaultListModel) queueList.getModel(); + int currentIndex = queueList.locationToIndex(e.getPoint()); + if (currentIndex != dragSourceIndex) { + int dragTargetIndex = queueList.getSelectedIndex(); + dragTargetIndex = Math.max(0, dragTargetIndex); + dragTargetIndex = Math.min(queueListModel.size() - 1, dragTargetIndex); + Object dragElement = queueListModel.get(dragSourceIndex); + queueListModel.remove(dragSourceIndex); + queueListModel.add(dragTargetIndex, dragElement); + dragSourceIndex = currentIndex; + } + } } @SuppressWarnings("unchecked") private void checkPopupTrigger(MouseEvent e) { - if (e.getModifiersEx() == InputEvent.BUTTON3_DOWN_MASK) { + if (SwingUtilities.isRightMouseButton(e)) { if (!(e.getSource() instanceof JList)) { return; } diff --git a/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java b/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java index f589e9dbb..3d4c1644f 100644 --- a/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java +++ b/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java @@ -16,6 +16,7 @@ public enum STATUS { DOWNLOAD_SKIP("Download Skipped"), TOTAL_BYTES("Total bytes"), COMPLETED_BYTES("Completed bytes"), + CHUNK_BYTES("Transferred bytes in last chunk"), RIP_ERRORED("Rip Errored"), NO_ALBUM_OR_USER("No album or user"); diff --git a/src/main/java/com/rarchives/ripme/uiUtils/ContextActionProtections.java b/src/main/java/com/rarchives/ripme/uiUtils/ContextActionProtections.java index 9237fea90..57ea0aa1b 100644 --- a/src/main/java/com/rarchives/ripme/uiUtils/ContextActionProtections.java +++ b/src/main/java/com/rarchives/ripme/uiUtils/ContextActionProtections.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.uiUtils; -import javax.swing.*; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import javax.swing.text.JTextComponent; import java.awt.*; import java.awt.datatransfer.Clipboard; @@ -10,6 +12,8 @@ import java.io.IOException; public class ContextActionProtections { + private static final Logger logger = LogManager.getLogger(ContextActionProtections.class); + public static void pasteFromClipboard(JTextComponent textComponent) { Clipboard clipboard = Toolkit.getDefaultToolkit().getSystemClipboard(); Transferable transferable = clipboard.getContents(new Object()); @@ -24,8 +28,8 @@ public static void pasteFromClipboard(JTextComponent textComponent) { // } // Set the text in the JTextField textComponent.setText(clipboardContent); - } catch (UnsupportedFlavorException | IOException unable_to_modify_text_on_paste) { - unable_to_modify_text_on_paste.printStackTrace(); + } catch (UnsupportedFlavorException | IOException e) { + logger.error("Unable to paste from clipboard", e); } } } diff --git a/src/main/java/com/rarchives/ripme/utils/DebouncedRunnable.java b/src/main/java/com/rarchives/ripme/utils/DebouncedRunnable.java new file mode 100644 index 000000000..135bffdce --- /dev/null +++ b/src/main/java/com/rarchives/ripme/utils/DebouncedRunnable.java @@ -0,0 +1,79 @@ +package com.rarchives.ripme.utils; + +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +public class DebouncedRunnable implements Runnable, AutoCloseable { + private final ScheduledExecutorService scheduler; + private final Runnable task; + private final long maxDelayMs; + private final AtomicLong lastInvoke = new AtomicLong(); + private final AtomicLong lastRun = new AtomicLong(); + private volatile ScheduledFuture future; + private final AtomicLong counter = new AtomicLong(0); + private final AtomicBoolean active = new AtomicBoolean(false); + private final Thread shutdownHook; + private volatile boolean closed = false; + + /** + * Debounce a task. + * @param task The task to run + * @param maxDelayMs Maximum delay to wait between calls in milliseconds + */ + public DebouncedRunnable(Runnable task, long maxDelayMs) { + this.task = task; + this.maxDelayMs = maxDelayMs; + this.scheduler = Executors.newSingleThreadScheduledExecutor(Thread.ofVirtual().factory()); + this.shutdownHook = new Thread(this::shutdown); + Runtime.getRuntime().addShutdownHook(shutdownHook); + } + + public void run() { + long now = System.currentTimeMillis(); + long timeSinceLastRun = now - lastRun.getAndSet(now); + boolean overMaxDelaySinceLastRun = timeSinceLastRun >= maxDelayMs; + if (overMaxDelaySinceLastRun) { + invoke(); + return; + } + long timeSinceLastInvoke = now - lastInvoke.get(); + boolean underMaxDelaySinceLastInvoke = timeSinceLastInvoke < maxDelayMs; + if (underMaxDelaySinceLastInvoke && future != null) { + future.cancel(false); + } + future = scheduler.schedule(this::invoke, maxDelayMs - timeSinceLastInvoke, TimeUnit.MILLISECONDS); + + } + + private void invoke() { + try { + active.set(true); + task.run(); + } finally { + lastInvoke.set(System.currentTimeMillis()); + active.set(false); + } + } + + public void shutdown() { + closed = true; + scheduler.shutdown(); + if (future != null && !future.isDone() && !active.get()) { + // Shutting down, but we still have a scheduled thread + future.cancel(false); + invoke(); + } + try { + Runtime.getRuntime().removeShutdownHook(shutdownHook); + } catch (IllegalStateException e) { + // The shutdown has already begun + } + } + + public void close() throws Exception { + if (!closed) { + shutdown(); + } + } +} diff --git a/src/main/java/com/rarchives/ripme/utils/Http.java b/src/main/java/com/rarchives/ripme/utils/Http.java index a1705f5a9..559f60e58 100644 --- a/src/main/java/com/rarchives/ripme/utils/Http.java +++ b/src/main/java/com/rarchives/ripme/utils/Http.java @@ -201,6 +201,10 @@ public Response response() throws IOException { IOException lastException = null; int retries = this.retries; while (--retries >= 0) { + // TODO uncomment and fix tests + //if (!MainWindow.isRipping.get()) { + // throw new IOException("Rip stopped, not making http request"); + //} try { response = connection.execute(); return response; diff --git a/src/main/java/com/rarchives/ripme/utils/TransferRate.java b/src/main/java/com/rarchives/ripme/utils/TransferRate.java new file mode 100644 index 000000000..c41d4296f --- /dev/null +++ b/src/main/java/com/rarchives/ripme/utils/TransferRate.java @@ -0,0 +1,79 @@ +package com.rarchives.ripme.utils; + +import java.util.Deque; +import java.util.concurrent.ConcurrentLinkedDeque; + +public class TransferRate { + private final Deque transferQueue = new ConcurrentLinkedDeque<>(); + private int windowDurationMs = 10000; + + public void addChunk(long bytes) { + long now = System.currentTimeMillis(); + transferQueue.addFirst(new ChunkStamp(now, bytes)); + removeOldChunks(now); + } + + public double calculateBytesPerSecond() { + if (transferQueue.isEmpty()) { + return 0; + } + long totalBytes = 0; + ChunkStamp oldest = transferQueue.getLast(); + long now = System.currentTimeMillis(); + removeOldChunks(now); + if (transferQueue.isEmpty()) { + return 0; + } + for (ChunkStamp chunkStamp : transferQueue) { + totalBytes += chunkStamp.bytes; + } + long elapsedMs = now - oldest.timestampMs; + double elapsedSeconds = (double) elapsedMs / 1000; + if (elapsedSeconds <= 0) { + return 0; + } + return totalBytes / elapsedSeconds; + } + + public String formatHumanTransferRate() { + double bps = calculateBytesPerSecond(); + int giB = 1024 * 1024 * 1024; + int miB = 1024 * 1024; + int kiB = 1024; + + // Format the string for less than 4 integer part digits if possible + if (bps < 1000) { + return String.format("%.2f B/s", bps); + } else if (bps / kiB < 1000) { + return String.format("%.2f KiB/s", bps / kiB); + } else if (bps / miB < 1000) { + return String.format("%.2f MiB/s", bps / miB); + } else { + return String.format("%.2f GiB/s", bps / giB); + } + } + + public void setWindowDurationMs(int windowDurationMs) { + this.windowDurationMs = windowDurationMs; + } + + public int getWindowDurationMs() { + return windowDurationMs; + } + + private void removeOldChunks(long now) { + while (!transferQueue.isEmpty() && transferQueue.getLast().timestampMs < now - windowDurationMs) { + transferQueue.removeLast(); + } + } + + private static class ChunkStamp { + long bytes; + long timestampMs; // millis + + ChunkStamp(long timestampMs, long bytes) { + this.bytes = bytes; + this.timestampMs = timestampMs; + } + } +} diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 6edc83785..802e28577 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -1,5 +1,6 @@ package com.rarchives.ripme.utils; +import java.awt.Desktop; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -15,6 +16,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.text.MessageFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -38,9 +40,11 @@ import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.appender.ConsoleAppender; import org.apache.logging.log4j.core.appender.RollingFileAppender; import org.apache.logging.log4j.core.appender.rolling.DefaultRolloverStrategy; import org.apache.logging.log4j.core.appender.rolling.SizeBasedTriggeringPolicy; @@ -49,6 +53,10 @@ import org.apache.logging.log4j.core.config.LoggerConfig; import com.rarchives.ripme.ripper.AbstractRipper; +import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilder; +import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilderFactory; +import org.apache.logging.log4j.core.config.builder.api.RootLoggerComponentBuilder; +import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration; /** * Common utility functions used in various places throughout the project. @@ -191,7 +199,12 @@ public static void setConfigList(String key, Enumeration enumeration) { public static void saveConfig() { try { - config.save(getConfigFilePath()); + // Simple hack: saveConfig is called from debouncedSaveConfig. + // Clone the config so we don't get a ConcurrentModificationException + // on config.save() if the config is updated between the time + // debounceSavedConfig.run() is called and the time saveConfig() is called. + PropertiesConfiguration clone = (PropertiesConfiguration) config.clone(); + clone.save(getConfigFilePath()); LOGGER.info("Saved configuration to " + getConfigFilePath()); } catch (ConfigurationException e) { LOGGER.error("Error while saving configuration: ", e); @@ -532,7 +545,7 @@ public static String getOriginalDirectory(String path) throws IOException { * @param bytes Non-human readable integer. * @return Human readable interpretation of a byte. */ - public static String bytesToHumanReadable(int bytes) { + public static String bytesToHumanReadable(long bytes) { float fbytes = (float) bytes; String[] mags = new String[]{"", "K", "M", "G", "T"}; int magIndex = 0; @@ -592,36 +605,45 @@ public static void playSound(String filename) { } } + public static void configureLogger() { + configureLogger(Level.INFO); // default INFO level + } + /** * Configures root logger, either for FILE output or just console. */ - public static void configureLogger() { - LoggerContext ctx = (LoggerContext) LogManager.getContext(false); - Configuration config = ctx.getConfiguration(); - LoggerConfig loggerConfig = config.getLoggerConfig(LogManager.ROOT_LOGGER_NAME); + public static void configureLogger(Level level) { + ConfigurationBuilder builder = ConfigurationBuilderFactory.newConfigurationBuilder(); + + //builder.setStatusLevel(Level.DEBUG); + final String consoleAppenderName = "stdout"; + builder.add(builder.newAppender(consoleAppenderName, "CONSOLE") + .addAttribute("target", ConsoleAppender.Target.SYSTEM_OUT) + .add(builder.newLayout("PatternLayout").addAttribute("pattern", "%-5level %c{1}: %msg%n%xEx")) + ); + + RootLoggerComponentBuilder rootLogger = builder.newRootLogger(level); + rootLogger.add(builder.newAppenderRef(consoleAppenderName)); // write to ripme.log file if checked in GUI boolean logSave = getConfigBoolean("log.save", false); if (logSave) { - LOGGER.debug("add rolling appender ripmelog"); - TriggeringPolicy tp = SizeBasedTriggeringPolicy.createPolicy("20M"); - DefaultRolloverStrategy rs = DefaultRolloverStrategy.newBuilder().withMax("2").build(); - RollingFileAppender rolling = RollingFileAppender.newBuilder() - .setName("ripmelog") - .withFileName("ripme.log") - .withFilePattern("%d{yyyy-MM-dd HH:mm:ss} %p %m%n") - .withPolicy(tp) - .withStrategy(rs) - .build(); - loggerConfig.addAppender(rolling, null, null); - } else { - LOGGER.debug("remove rolling appender ripmelog"); - if (config.getAppender("ripmelog") != null) { - config.getAppender("ripmelog").stop(); - } - loggerConfig.removeAppender("ripmelog"); + final String fileAppenderName = "rolling"; + builder.add(builder.newAppender(fileAppenderName, "RollingFile") + .addAttribute("fileName", "ripme.log") + .addAttribute("filePattern", "ripme-%d{yyyy-MM-dd}-%i.log.gz") + .add(builder.newLayout("PatternLayout").addAttribute("pattern", "%d %-5level %c{1}: %msg%n%xEx")) + .addComponent(builder.newComponent("Policies") + .addComponent(builder.newComponent("SizeBasedTriggeringPolicy").addAttribute("size", "20M"))) + ); + rootLogger.add(builder.newAppenderRef(fileAppenderName)); } - ctx.updateLoggers(); // This causes all Loggers to refetch information from their LoggerConfig. + + builder.add(rootLogger); + + Configuration configuration = builder.build(); + LoggerContext ctx = (LoggerContext) LogManager.getContext(false); + ctx.reconfigure(configuration); } /** @@ -798,9 +820,16 @@ public static String[] getSupportedLanguages() { } public static String getLocalizedString(String key) { - LOGGER.debug(String.format("Key %s in %s is: %s", key, getSelectedLanguage(), - resourceBundle.getString(key))); - return resourceBundle.getString(key); + String message = resourceBundle.getString(key); + LOGGER.trace("Key {} in {} is: {}", key, getSelectedLanguage(), message); + return message; + } + + @SuppressWarnings({"JavaExistingMethodCanBeUsed", "LoggingSimilarMessage"}) + public static String getLocalizedString(String key, Object... args) { + String pattern = resourceBundle.getString(key); + LOGGER.trace("Key {} in {} is: {}", key, getSelectedLanguage(), pattern); + return MessageFormat.format(pattern, args); } /** @@ -876,4 +905,53 @@ public static void sleep(long time) { e1.printStackTrace(); } } + + /** + * Open a File using the default OS handler, but not in a child process, allowing the program to exit without closing all opened windows. + *
+ * In comparison, {@link Desktop#open(File)} opens the file manager or browser in a child process, blocking the JVM from exiting. + * @param file The File to open. + */ + public static void open(File file) throws IOException { + if (isUnix() && which("nohup") && which("xdg-open")) { + linuxOpen(file.toURI()); + return; + } + Desktop.getDesktop().open(file); + } + + /** + * Open a URI using the default OS handler, but not in a child process, allowing the program to exit without closing all opened windows. + *
+ * In comparison, {@link Desktop#open(File)} opens the file manager or browser in a child process, blocking the JVM from exiting. + * @param uri The URI to open. + */ + public static void browse(URI uri) throws IOException { + if (isUnix() && which("nohup") && which("xdg-open")) { + linuxOpen(uri); + return; + } + Desktop.getDesktop().browse(uri); + } + + private static void linuxOpen(URI uri) throws IOException { + new ProcessBuilder("nohup", "xdg-open", uri.toString()) + .redirectOutput(ProcessBuilder.Redirect.DISCARD) + .redirectError(ProcessBuilder.Redirect.DISCARD) + .start(); + } + + public static boolean which(String command) { + String pathEnv = System.getenv("PATH"); + if (isWindows()) { + command = command + ".exe"; + } + for (String dir : pathEnv.split(File.pathSeparator)) { + File file = new File(dir, command); + if (file.isFile() && file.canExecute()) { + return true; + } + } + return false; + } } diff --git a/src/main/resources/LabelsBundle.properties b/src/main/resources/LabelsBundle.properties index 6a48b245e..8648ca91b 100644 --- a/src/main/resources/LabelsBundle.properties +++ b/src/main/resources/LabelsBundle.properties @@ -1,3 +1,7 @@ +# Buttons +Rip = Rip +Stop = Stop +Panic = Panic! Log = Log History = History created = created @@ -6,6 +10,14 @@ queue = Queue Configuration = Configuration open = Open +# Status labels +Pending = Pending +Active = Active +Completed = Completed +Errored = Errored +Total = Total +Speed = Speed + # Keys for the Configuration menu current.version = Current version check.for.updates = Check for updates @@ -31,8 +43,11 @@ loading.history.from = Loading history from # Queue keys queue.remove.all = Remove All -queue.validation = Are you sure you want to remove all elements from the queue? +queue.remove.all.validation = Are you sure you want to remove all elements from the queue? queue.remove.selected = Remove Selected +queue.remove.selected.validation = Are you sure you want to remove the selected elements from the queue? +queue.move.selected.to.top = Move Selected to Top +queue.move.selected.to.bottom = Move Selected to Bottom # History re-rip.checked = Re-rip Checked @@ -59,6 +74,8 @@ inactive = Inactive download.url.list = Download url list select.save.dir = Select Save Directory +attempting.to.start.rip.for.album.0 = Attempting to start rip for album {0} + # Keys for the logs generated by DownloadFileThread nonretriable.status.code = Non-retriable status code retriable.status.code = Retriable status code @@ -69,9 +86,18 @@ magic.number.was = Magic number was deleting.existing.file = Deleting existing file request.properties = Request properties download.interrupted = Download interrupted +rip.interrupted = Rip interrupted +rip.gracefully.stopping = Rip gracefully stopping exceeded.maximum.retries = Exceeded maximum retries http.status.exception = HTTP status exception exception.while.downloading.file = Exception while downloading file failed.to.download = Failed to download skipping = Skipping -file.already.exists = file already exists \ No newline at end of file +file.already.exists = file already exists +skipping.ignored.extension = Skipping ignored extension + +failed.to.get.url.for.0 = Failed to get URL for {0} +0.already.saved.as.1 = {0} already saved as {1} +error.creating.directory = Error creating directory +0.while.downloading.1 = {0} while downloading {1} +no.space.left.on.device = No space left on device diff --git a/src/main/resources/LabelsBundle_el_GR.properties b/src/main/resources/LabelsBundle_el_GR.properties index 14656e877..07c61900f 100644 --- a/src/main/resources/LabelsBundle_el_GR.properties +++ b/src/main/resources/LabelsBundle_el_GR.properties @@ -29,7 +29,7 @@ loading.history.from = Φόρτωση ιστορικού από # Queue keys queue.remove.all = Διαγραφή όλων -queue.validation = Είσαι σίγουρος οτι θέλεις να διαγράφουν όλα τα στοιχεια της ουράς? +queue.remove.all.validation = Είσαι σίγουρος οτι θέλεις να διαγράφουν όλα τα στοιχεια της ουράς? queue.remove.selected = Διαγραφή επιλεγμένου # History @@ -72,4 +72,4 @@ http.status.exception = HTTP status λάθος exception.while.downloading.file = Λάθος ενω μεταφορτώνοταν ενα αρχειο failed.to.download = Αποτυχία μεταφόρτωσης skipping = Παράκαμψη -file.already.exists = το αρχείο υπάρχει ήδη \ No newline at end of file +file.already.exists = το αρχείο υπάρχει ήδη diff --git a/src/main/resources/LabelsBundle_es_ES.properties b/src/main/resources/LabelsBundle_es_ES.properties index fea84e5d5..96f170dbf 100644 --- a/src/main/resources/LabelsBundle_es_ES.properties +++ b/src/main/resources/LabelsBundle_es_ES.properties @@ -30,7 +30,7 @@ loading.history.from = Cargando historia desde # Queue keys queue.remove.all = Eliminar todos los elementos -queue.validation = ¿Está seguro que desea eliminar todos los elementos de la lista? +queue.remove.all.validation = ¿Está seguro que desea eliminar todos los elementos de la lista? queue.remove.selected = Eliminar elementos seleccionados # History @@ -73,4 +73,4 @@ http.status.exception = Error de estado HTTP exception.while.downloading.file = Error al descargar archivo failed.to.download = Descarga fallida skipping = Saltando -file.already.exists = el fichero ya existe \ No newline at end of file +file.already.exists = el fichero ya existe diff --git a/src/main/resources/LabelsBundle_pt_PT.properties b/src/main/resources/LabelsBundle_pt_PT.properties index 500049ce9..dba5adca3 100644 --- a/src/main/resources/LabelsBundle_pt_PT.properties +++ b/src/main/resources/LabelsBundle_pt_PT.properties @@ -29,7 +29,7 @@ loading.history.from = Carregar histórico de # Queue keys queue.remove.all = Remover todos -queue.validation = Tem a certeza de que quer remover todos os elementos da fila? +queue.remove.all.validation = Tem a certeza de que quer remover todos os elementos da fila? queue.remove.selected = Remover seleccionados # History @@ -72,4 +72,4 @@ http.status.exception = Exceção de status HTTP exception.while.downloading.file = Exceção enquanto o ficheiro era baixado failed.to.download = Falha no download skipping = Pulando -file.already.exists = Ficheiro já existe \ No newline at end of file +file.already.exists = Ficheiro já existe diff --git a/src/main/resources/LabelsBundle_zh_CN.properties b/src/main/resources/LabelsBundle_zh_CN.properties index 7cf6d7810..c7865cc44 100644 --- a/src/main/resources/LabelsBundle_zh_CN.properties +++ b/src/main/resources/LabelsBundle_zh_CN.properties @@ -29,7 +29,7 @@ loading.history.from = 加载历史从 # Queue keys queue.remove.all = 移除全部 -queue.validation = 您确定要移除队列内的全部项目？ +queue.remove.all.validation = 您确定要移除队列内的全部项目？ queue.remove.selected = 移除所选项目 # History @@ -72,4 +72,4 @@ http.status.exception = HTTP 状态意外 exception.while.downloading.file = 下载文件时发生意外 failed.to.download = 下载失败 skipping = 跳过 -file.already.exists = 文件已存在 \ No newline at end of file +file.already.exists = 文件已存在 diff --git a/src/test/java/com/rarchives/ripme/utils/TransferRateTest.java b/src/test/java/com/rarchives/ripme/utils/TransferRateTest.java new file mode 100644 index 000000000..66d1463cc --- /dev/null +++ b/src/test/java/com/rarchives/ripme/utils/TransferRateTest.java @@ -0,0 +1,35 @@ +package com.rarchives.ripme.utils; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +class TransferRateTest { + + @Test + void formatHumanTransferRate() { + //TransferRate transferRate = new TransferRate(); + TransferRate transferRate = Mockito.spy(); + + Mockito.when(transferRate.calculateBytesPerSecond()).thenReturn(999.); + Assertions.assertEquals("999.00 B/s", transferRate.formatHumanTransferRate()); + + Mockito.when(transferRate.calculateBytesPerSecond()).thenReturn(1013.); + Assertions.assertEquals("0.99 KiB/s", transferRate.formatHumanTransferRate()); + + Mockito.when(transferRate.calculateBytesPerSecond()).thenReturn(1024.0 * 1024); + Assertions.assertEquals("1.00 MiB/s", transferRate.formatHumanTransferRate()); + + Mockito.when(transferRate.calculateBytesPerSecond()).thenReturn(1024.0 * 1013); + Assertions.assertEquals("0.99 MiB/s", transferRate.formatHumanTransferRate()); + + Mockito.when(transferRate.calculateBytesPerSecond()).thenReturn(1024.0 * 1024 * 1024); + Assertions.assertEquals("1.00 GiB/s", transferRate.formatHumanTransferRate()); + + Mockito.when(transferRate.calculateBytesPerSecond()).thenReturn(1024.0 * 1024 * 1013); + Assertions.assertEquals("0.99 GiB/s", transferRate.formatHumanTransferRate()); + + Mockito.when(transferRate.calculateBytesPerSecond()).thenReturn(1024.0 * 1024 * 1024 * 1013); + Assertions.assertEquals("1013.00 GiB/s", transferRate.formatHumanTransferRate()); + } +}