diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index bd373a4e0..0c2be7c71 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -111,7 +111,7 @@ private static void rip(URL url) throws Exception { entry.url = u; entry.dir = ripper.getWorkingDir().getAbsolutePath(); try { - entry.title = ripper.getAlbumTitle(ripper.getURL()); + entry.title = ripper.getAlbumTitle(); } catch (MalformedURLException ignored) { } HISTORY.add(entry); } diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 0740f62c4..2ad4dc912 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -8,37 +8,23 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.jsoup.nodes.Document; import com.rarchives.ripme.ui.MainWindow; -import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; +import com.rarchives.ripme.utils.RipUtils; /** * Simplified ripper, designed for ripping from sites by parsing HTML. */ -public abstract class AbstractHTMLRipper extends AbstractRipper { +public abstract class AbstractHTMLRipper extends QueueingRipper { - private static final Logger logger = LogManager.getLogger(AbstractHTMLRipper.class); - - private final Map itemsPending = Collections.synchronizedMap(new HashMap<>()); - private final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); - private final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); Document cachedFirstPage; protected AbstractHTMLRipper(URL url) throws IOException { @@ -68,6 +54,10 @@ public Document getNextPage(Document doc) throws IOException, URISyntaxException return null; } + public List getURLsFromFirstPage() throws UnsupportedEncodingException, IOException, URISyntaxException { + return RipUtils.toURLList(getURLsFromPage(getCachedFirstPage())); + } + protected abstract List getURLsFromPage(Document page) throws UnsupportedEncodingException, URISyntaxException; protected List getDescriptionsFromPage(Document doc) throws IOException { @@ -115,7 +105,7 @@ protected boolean hasQueueSupport() { } // Takes a url and checks if it is for a page of albums - protected boolean pageContainsAlbums(URL url) { + protected boolean pageContainsAlbums() { return false; } @@ -127,7 +117,7 @@ public void rip() throws IOException, URISyntaxException { sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm()); var doc = getCachedFirstPage(); - if (hasQueueSupport() && pageContainsAlbums(this.url)) { + if (hasQueueSupport() && pageContainsAlbums()) { List urls = getAlbumsToQueue(doc); for (String url : urls) { MainWindow.addUrlToQueue(url); @@ -329,156 +319,6 @@ protected String getPrefix(int index) { return prefix; } - /* - * ------ Methods copied from AlbumRipper. ------ - * This removes AlbumnRipper's usage from this class. - */ - - protected boolean allowDuplicates() { - return false; - } - - @Override - /* - Returns total amount of files attempted. - */ - public int getCount() { - return itemsCompleted.size() + itemsErrored.size(); - } - - @Override - /* - Queues multiple URLs of single images to download from a single Album URL - */ - public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { - // Only download one file if this is a test. - if (isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { - stop(); - itemsPending.clear(); - return false; - } - if (!allowDuplicates() - && ( itemsPending.containsKey(url) - || itemsCompleted.containsKey(url) - || itemsErrored.containsKey(url) )) { - // Item is already downloaded/downloading, skip it. - logger.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs)); - return false; - } - if (shouldIgnoreURL(url)) { - sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); - return false; - } - if (Utils.getConfigBoolean("urls_only.save", false)) { - // Output URL to file - Path urlFile = Paths.get(this.workingDir + "/urls.txt"); - String text = url.toExternalForm() + System.lineSeparator(); - try { - Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND); - itemsCompleted.put(url, urlFile); - } catch (IOException e) { - logger.error("Error while writing to " + urlFile, e); - } - } - else { - itemsPending.put(url, saveAs.toFile()); - DownloadFileThread dft = new DownloadFileThread(url, saveAs.toFile(), this, getFileExtFromMIME); - if (referrer != null) { - dft.setReferrer(referrer); - } - if (cookies != null) { - dft.setCookies(cookies); - } - threadPool.addThread(dft); - } - - return true; - } - - @Override - public boolean addURLToDownload(URL url, Path saveAs) { - return addURLToDownload(url, saveAs, null, null, false); - } - - /** - * Queues image to be downloaded and saved. - * Uses filename from URL to decide filename. - * @param url - * URL to download - * @return - * True on success - */ - protected boolean addURLToDownload(URL url) { - // Use empty prefix and empty subdirectory - return addURLToDownload(url, "", ""); - } - - @Override - /* - Cleans up & tells user about successful download - */ - public void downloadCompleted(URL url, Path saveAs) { - if (observer == null) { - return; - } - try { - String path = Utils.removeCWD(saveAs); - RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); - itemsPending.remove(url); - itemsCompleted.put(url, saveAs); - observer.update(this, msg); - - checkIfComplete(); - } catch (Exception e) { - logger.error("Exception while updating observer: ", e); - } - } - - @Override - /* - * Cleans up & tells user about failed download. - */ - public void downloadErrored(URL url, String reason) { - if (observer == null) { - return; - } - itemsPending.remove(url); - itemsErrored.put(url, reason); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason)); - - checkIfComplete(); - } - - @Override - /* - Tells user that a single file in the album they wish to download has - already been downloaded in the past. - */ - public void downloadExists(URL url, Path file) { - if (observer == null) { - return; - } - - itemsPending.remove(url); - itemsCompleted.put(url, file); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " already saved as " + file)); - - checkIfComplete(); - } - - /** - * Notifies observers and updates state if all files have been ripped. - */ - @Override - protected void checkIfComplete() { - if (observer == null) { - return; - } - if (itemsPending.isEmpty()) { - super.checkIfComplete(); - } - } - /** * Sets directory to save all ripped files to. * @param url @@ -492,7 +332,7 @@ public void setWorkingDir(URL url) throws IOException, URISyntaxException { if (!path.endsWith(File.separator)) { path += File.separator; } - String title = getAlbumTitle(this.url); + String title = getAlbumTitle(); logger.debug("Using album title '" + title + "'"); title = Utils.filesystemSafe(title); @@ -509,28 +349,4 @@ public void setWorkingDir(URL url) throws IOException, URISyntaxException { logger.debug("Set working directory to: " + this.workingDir); } - /** - * @return - * Integer between 0 and 100 defining the progress of the album rip. - */ - @Override - public int getCompletionPercentage() { - double total = itemsPending.size() + itemsErrored.size() + itemsCompleted.size(); - return (int) (100 * ( (total - itemsPending.size()) / total)); - } - - /** - * @return - * Human-readable information on the status of the current rip. - */ - @Override - public String getStatusText() { - return getCompletionPercentage() + - "% " + - "- Pending: " + itemsPending.size() + - ", Completed: " + itemsCompleted.size() + - ", Errored: " + itemsErrored.size(); - } - - } diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index a49084c63..3884e3a91 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -1,39 +1,24 @@ package com.rarchives.ripme.ripper; -import java.io.File; import java.io.IOException; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; -import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.json.JSONObject; -import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; +import com.rarchives.ripme.utils.RipUtils; import com.rarchives.ripme.utils.Utils; /** * Simplified ripper, designed for ripping from sites by parsing JSON. */ -public abstract class AbstractJSONRipper extends AbstractRipper { - - private static final Logger logger = LogManager.getLogger(AbstractJSONRipper.class); - - private Map itemsPending = Collections.synchronizedMap(new HashMap()); - private Map itemsCompleted = Collections.synchronizedMap(new HashMap()); - private Map itemsErrored = Collections.synchronizedMap(new HashMap()); +public abstract class AbstractJSONRipper extends QueueingRipper { protected AbstractJSONRipper(URL url) throws IOException { super(url); @@ -124,6 +109,10 @@ public void rip() throws IOException, URISyntaxException { waitForThreads(); } + public List getFirstPageURLs() throws IOException, URISyntaxException { + return RipUtils.toURLList(getURLsFromJSON(getFirstPage())); + } + protected String getPrefix(int index) { String prefix = ""; if (keepSortOrder() && Utils.getConfigBoolean("download.save_order", true)) { @@ -132,155 +121,6 @@ protected String getPrefix(int index) { return prefix; } - /* - * ------ Methods copied from AlbumRipper ------ - */ - - protected boolean allowDuplicates() { - return false; - } - - @Override - /** - * Returns total amount of files attempted. - */ - public int getCount() { - return itemsCompleted.size() + itemsErrored.size(); - } - - @Override - /** - * Queues multiple URLs of single images to download from a single Album URL - */ - public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { - // Only download one file if this is a test. - if (super.isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { - stop(); - itemsPending.clear(); - return false; - } - if (!allowDuplicates() - && ( itemsPending.containsKey(url) - || itemsCompleted.containsKey(url) - || itemsErrored.containsKey(url) )) { - // Item is already downloaded/downloading, skip it. - logger.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs)); - return false; - } - if (shouldIgnoreURL(url)) { - sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); - return false; - } - if (Utils.getConfigBoolean("urls_only.save", false)) { - // Output URL to file - Path urlFile = Paths.get(this.workingDir + "/urls.txt"); - String text = url.toExternalForm() + System.lineSeparator(); - try { - Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND); - itemsCompleted.put(url, urlFile); - } catch (IOException e) { - logger.error("Error while writing to " + urlFile, e); - } - } - else { - itemsPending.put(url, saveAs.toFile()); - DownloadFileThread dft = new DownloadFileThread(url, saveAs.toFile(), this, getFileExtFromMIME); - if (referrer != null) { - dft.setReferrer(referrer); - } - if (cookies != null) { - dft.setCookies(cookies); - } - threadPool.addThread(dft); - } - - return true; - } - - @Override - public boolean addURLToDownload(URL url, Path saveAs) { - return addURLToDownload(url, saveAs, null, null, false); - } - - /** - * Queues image to be downloaded and saved. - * Uses filename from URL to decide filename. - * @param url - * URL to download - * @return - * True on success - */ - protected boolean addURLToDownload(URL url) { - // Use empty prefix and empty subdirectory - return addURLToDownload(url, "", ""); - } - - @Override - /** - * Cleans up & tells user about successful download - */ - public void downloadCompleted(URL url, Path saveAs) { - if (observer == null) { - return; - } - try { - String path = Utils.removeCWD(saveAs); - RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); - itemsPending.remove(url); - itemsCompleted.put(url, saveAs); - observer.update(this, msg); - - checkIfComplete(); - } catch (Exception e) { - logger.error("Exception while updating observer: ", e); - } - } - - @Override - /** - * Cleans up & tells user about failed download. - */ - public void downloadErrored(URL url, String reason) { - if (observer == null) { - return; - } - itemsPending.remove(url); - itemsErrored.put(url, reason); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason)); - - checkIfComplete(); - } - - @Override - /** - * Tells user that a single file in the album they wish to download has - * already been downloaded in the past. - */ - public void downloadExists(URL url, Path file) { - if (observer == null) { - return; - } - - itemsPending.remove(url); - itemsCompleted.put(url, file); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " already saved as " + file)); - - checkIfComplete(); - } - - /** - * Notifies observers and updates state if all files have been ripped. - */ - @Override - protected void checkIfComplete() { - if (observer == null) { - return; - } - if (itemsPending.isEmpty()) { - super.checkIfComplete(); - } - } - /** * Sets directory to save all ripped files to. * @param url @@ -293,9 +133,9 @@ public void setWorkingDir(URL url) throws IOException, URISyntaxException { Path wd = Utils.getWorkingDirectory(); String title; if (Utils.getConfigBoolean("album_titles.save", true)) { - title = getAlbumTitle(this.url); + title = getAlbumTitle(); } else { - title = super.getAlbumTitle(this.url); + title = super.getAlbumTitle(); } logger.debug("Using album title '" + title + "'"); @@ -308,31 +148,4 @@ public void setWorkingDir(URL url) throws IOException, URISyntaxException { this.workingDir = wd.toFile(); logger.info("Set working directory to: {}", this.workingDir); } - - /** - * @return - * Integer between 0 and 100 defining the progress of the album rip. - */ - @Override - public int getCompletionPercentage() { - double total = itemsPending.size() + itemsErrored.size() + itemsCompleted.size(); - return (int) (100 * ( (total - itemsPending.size()) / total)); - } - - /** - * @return - * Human-readable information on the status of the current rip. - */ - @Override - public String getStatusText() { - StringBuilder sb = new StringBuilder(); - sb.append(getCompletionPercentage()) - .append("% ") - .append("- Pending: " ).append(itemsPending.size()) - .append(", Completed: ").append(itemsCompleted.size()) - .append(", Errored: " ).append(itemsErrored.size()); - return sb.toString(); - } - - } diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 8ccff6481..d1ba56190 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -215,7 +215,13 @@ public void setup() throws IOException, URISyntaxException { // ctx.reconfigure(); // ctx.updateLoggers(); - this.threadPool = new DownloadThreadPool(); + initThreadPool(); + } + + public synchronized void initThreadPool() { + if (threadPool == null) { + this.threadPool = new DownloadThreadPool(); + } } public void setObserver(RipStatusHandler obs) { @@ -481,7 +487,9 @@ public static String getFileName(URL url, String prefix, String fileName, String protected void waitForThreads() { logger.debug("Waiting for threads to finish"); completed = false; - threadPool.waitForThreads(); + if (threadPool != null) { + threadPool.waitForThreads(); + } checkIfComplete(); } @@ -576,17 +584,20 @@ public File getWorkingDir() { return workingDir; } + public void setWorkingDir(File dir) { + this.workingDir = dir; + } + @Override public abstract void setWorkingDir(URL url) throws IOException, URISyntaxException; /** - * @param url The URL you want to get the title of. * @return host_URLid * e.g. (for a reddit post) * reddit_post_7mg2ur * @throws MalformedURLException If any of those damned URLs gets malformed. */ - public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { + public String getAlbumTitle() throws MalformedURLException, URISyntaxException { try { return getHost() + "_" + getGID(url); } catch (URISyntaxException e) { diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index bda3bf6fb..c7711698f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -5,20 +5,8 @@ import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.rarchives.ripme.ui.RipStatusMessage; -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Utils; // Should this file even exist? It does the same thing as abstractHTML ripper @@ -27,14 +15,7 @@ * For ripping delicious albums off the interwebz. * @deprecated Use AbstractHTMLRipper instead. */ -@Deprecated -public abstract class AlbumRipper extends AbstractRipper { - - private static final Logger logger = LogManager.getLogger(AlbumRipper.class); - - private Map itemsPending = Collections.synchronizedMap(new HashMap()); - private Map itemsCompleted = Collections.synchronizedMap(new HashMap()); - private Map itemsErrored = Collections.synchronizedMap(new HashMap()); +public abstract class AlbumRipper extends QueueingRipper { protected AlbumRipper(URL url) throws IOException { super(url); @@ -46,151 +27,6 @@ protected AlbumRipper(URL url) throws IOException { public abstract String getHost(); public abstract String getGID(URL url) throws MalformedURLException, URISyntaxException; - protected boolean allowDuplicates() { - return false; - } - - @Override - /** - * Returns total amount of files attempted. - */ - public int getCount() { - return itemsCompleted.size() + itemsErrored.size(); - } - - @Override - /** - * Queues multiple URLs of single images to download from a single Album URL - */ - public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { - // Only download one file if this is a test. - if (super.isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { - stop(); - itemsPending.clear(); - return false; - } - if (!allowDuplicates() - && ( itemsPending.containsKey(url) - || itemsCompleted.containsKey(url) - || itemsErrored.containsKey(url) )) { - // Item is already downloaded/downloading, skip it. - logger.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs)); - return false; - } - if (shouldIgnoreURL(url)) { - sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); - return false; - } - if (Utils.getConfigBoolean("urls_only.save", false)) { - // Output URL to file - Path urlFile = Paths.get(this.workingDir + "/urls.txt"); - String text = url.toExternalForm() + System.lineSeparator(); - try { - Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND); - itemsCompleted.put(url, urlFile); - } catch (IOException e) { - logger.error("Error while writing to " + urlFile, e); - } - } - else { - itemsPending.put(url, saveAs.toFile()); - DownloadFileThread dft = new DownloadFileThread(url, saveAs.toFile(), this, getFileExtFromMIME); - if (referrer != null) { - dft.setReferrer(referrer); - } - if (cookies != null) { - dft.setCookies(cookies); - } - threadPool.addThread(dft); - } - - return true; - } - - @Override - public boolean addURLToDownload(URL url, Path saveAs) { - return addURLToDownload(url, saveAs, null, null, false); - } - - /** - * Queues image to be downloaded and saved. - * Uses filename from URL to decide filename. - * @param url - * URL to download - * @return - * True on success - */ - protected boolean addURLToDownload(URL url) { - // Use empty prefix and empty subdirectory - return addURLToDownload(url, "", ""); - } - - @Override - /** - * Cleans up & tells user about successful download - */ - public void downloadCompleted(URL url, Path saveAs) { - if (observer == null) { - return; - } - try { - String path = Utils.removeCWD(saveAs); - RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); - itemsPending.remove(url); - itemsCompleted.put(url, saveAs); - observer.update(this, msg); - - checkIfComplete(); - } catch (Exception e) { - logger.error("Exception while updating observer: ", e); - } - } - - @Override - /** - * Cleans up & tells user about failed download. - */ - public void downloadErrored(URL url, String reason) { - if (observer == null) { - return; - } - itemsPending.remove(url); - itemsErrored.put(url, reason); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason)); - - checkIfComplete(); - } - - @Override - /** - * Tells user that a single file in the album they wish to download has - * already been downloaded in the past. - */ - public void downloadExists(URL url, Path file) { - if (observer == null) { - return; - } - - itemsPending.remove(url); - itemsCompleted.put(url, file); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " already saved as " + file)); - - checkIfComplete(); - } - - /** - * Notifies observers and updates state if all files have been ripped. - */ - @Override - protected void checkIfComplete() { - if (observer == null) { - return; - } - if (itemsPending.isEmpty()) { - super.checkIfComplete(); - } - } - /** * Sets directory to save all ripped files to. * @param url @@ -208,9 +44,9 @@ public void setWorkingDir(URL url) throws IOException, URISyntaxException { } String title; if (Utils.getConfigBoolean("album_titles.save", true)) { - title = getAlbumTitle(this.url); + title = getAlbumTitle(); } else { - title = super.getAlbumTitle(this.url); + title = super.getAlbumTitle(); } logger.debug("Using album title '" + title + "'"); @@ -226,28 +62,4 @@ public void setWorkingDir(URL url) throws IOException, URISyntaxException { logger.debug("Set working directory to: " + this.workingDir); } - /** - * @return - * Integer between 0 and 100 defining the progress of the album rip. - */ - @Override - public int getCompletionPercentage() { - double total = itemsPending.size() + itemsErrored.size() + itemsCompleted.size(); - return (int) (100 * ( (total - itemsPending.size()) / total)); - } - - /** - * @return - * Human-readable information on the status of the current rip. - */ - @Override - public String getStatusText() { - StringBuilder sb = new StringBuilder(); - sb.append(getCompletionPercentage()) - .append("% ") - .append("- Pending: " ).append(itemsPending.size()) - .append(", Completed: ").append(itemsCompleted.size()) - .append(", Errored: " ).append(itemsErrored.size()); - return sb.toString(); - } } diff --git a/src/main/java/com/rarchives/ripme/ripper/QueueingRipper.java b/src/main/java/com/rarchives/ripme/ripper/QueueingRipper.java new file mode 100644 index 000000000..529855922 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/QueueingRipper.java @@ -0,0 +1,227 @@ +package com.rarchives.ripme.ripper; + +import java.io.IOException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.rarchives.ripme.ui.RipStatusMessage; +import com.rarchives.ripme.ui.RipStatusMessage.STATUS; +import com.rarchives.ripme.utils.Utils; + +public abstract class QueueingRipper extends AbstractRipper { + + protected final Logger logger = LogManager.getLogger(getClass()); + + private final Set itemsPending = Collections.synchronizedSet(new HashSet<>()); + private final Set itemsCompleted = Collections.synchronizedSet(new HashSet<>()); + private final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); + + public QueueingRipper(URL url) throws IOException { + super(url); + } + + /* + * ------ Methods copied from AlbumRipper. ------ + * This removes AlbumnRipper's usage from this class. + */ + protected boolean allowDuplicates() { + return false; + } + + @Override + /* + Returns total amount of files attempted. + */ + public int getCount() { + return itemsCompleted.size() + itemsErrored.size(); + } + + @Override + /* + Queues multiple URLs of single images to download from a single Album URL + */ + public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { + // Only download one file if this is a test. + if (isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { + stop(); + itemsPending.clear(); + return false; + } + if (!allowDuplicates() + && ( itemsPending.contains(url) + || itemsCompleted.contains(url) + || itemsErrored.containsKey(url) )) { + // Item is already downloaded/downloading, skip it. + logger.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs)); + return false; + } + if (shouldIgnoreURL(url)) { + sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); + return false; + } + if (Utils.getConfigBoolean("urls_only.save", false)) { + // Output URL to file + final var urlFile = Paths.get(this.workingDir + "/urls.txt"); + final var text = url.toExternalForm() + System.lineSeparator(); + try { + Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); + itemsCompleted.add(url); + } catch (final IOException e) { + logger.error("Error while writing to " + urlFile, e); + } + } + else { + addURLToPending(url); + downloadInBackground(url, saveAs, referrer, cookies, getFileExtFromMIME); + } + + return true; + } + + /** + * Start downloading of the given url in the background, using the internal threadpool. + * @param url + * @param saveAs + * @param referrer + * @param cookies + * @param getFileExtFromMIME + */ + protected void downloadInBackground(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { + final var dft = new DownloadFileThread(url, saveAs.toFile(), this, getFileExtFromMIME); + if (referrer != null) { + dft.setReferrer(referrer); + } + if (cookies != null) { + dft.setCookies(cookies); + } + threadPool.addThread(dft); + } + + @Override + public boolean addURLToDownload(URL url, Path saveAs) { + return addURLToDownload(url, saveAs, null, null, false); + } + + /** + * Queues image to be downloaded and saved. + * Uses filename from URL to decide filename. + * @param url + * URL to download + * @return + * True on success + */ + protected boolean addURLToDownload(URL url) { + // Use empty prefix and empty subdirectory + return addURLToDownload(url, "", ""); + } + + protected void addURLToPending(URL url) { + if (observer == null) { + return; + } + itemsPending.add(url); + observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_PROGRESSED, url)); + } + + @Override + /* + Cleans up & tells user about successful download + */ + public void downloadCompleted(URL url, Path saveAs) { + if (observer == null) { + return; + } + try { + final var path = Utils.removeCWD(saveAs); + final var msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); + itemsPending.remove(url); + itemsCompleted.add(url); + observer.update(this, msg); + + checkIfComplete(); + } catch (final Exception e) { + logger.error("Exception while updating observer: ", e); + } + } + + @Override + /* + * Cleans up & tells user about failed download. + */ + public void downloadErrored(URL url, String reason) { + if (observer == null) { + return; + } + itemsPending.remove(url); + itemsErrored.put(url, reason); + observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason)); + + checkIfComplete(); + } + + @Override + /* + Tells user that a single file in the album they wish to download has + already been downloaded in the past. + */ + public void downloadExists(URL url, Path file) { + if (observer == null) { + return; + } + + itemsPending.remove(url); + itemsCompleted.add(url); + observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " already saved as " + file)); + + checkIfComplete(); + } + + /** + * Notifies observers and updates state if all files have been ripped. + */ + @Override + protected void checkIfComplete() { + if (observer == null) { + return; + } + if (itemsPending.isEmpty()) { + super.checkIfComplete(); + } + } + + /** + * @return + * Integer between 0 and 100 defining the progress of the album rip. + */ + @Override + public int getCompletionPercentage() { + final double total = itemsPending.size() + itemsErrored.size() + itemsCompleted.size(); + return (int) (100 * ( (total - itemsPending.size()) / total)); + } + + /** + * @return + * Human-readable information on the status of the current rip. + */ + @Override + public String getStatusText() { + return getCompletionPercentage() + + "% " + + "- Pending: " + itemsPending.size() + + ", Completed: " + itemsCompleted.size() + + ", Errored: " + itemsErrored.size(); + } + +} diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index 785f3d92b..f8bd5e560 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -44,7 +44,7 @@ public void setBytesCompleted(int bytes) { } @Override - public String getAlbumTitle(URL url) { + public String getAlbumTitle() { return "videos"; } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/AllporncomicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/AllporncomicRipper.java index da8c7bd74..8ca344f93 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/AllporncomicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/AllporncomicRipper.java @@ -60,7 +60,7 @@ public boolean hasQueueSupport() { } @Override - public boolean pageContainsAlbums(URL url) { + public boolean pageContainsAlbums() { Pattern pa = Pattern.compile("^https?://allporncomic.com/porncomic/([a-zA-Z0-9_\\-]+)/?$"); Matcher ma = pa.matcher(url.toExternalForm()); return ma.matches(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/BatoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/BatoRipper.java index c1780818a..061b82e95 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/BatoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/BatoRipper.java @@ -58,7 +58,7 @@ public boolean hasQueueSupport() { } @Override - public boolean pageContainsAlbums(URL url) { + public boolean pageContainsAlbums() { Pattern p = Pattern.compile("https?://bato.to/series/([\\d]+)/?"); Matcher m = p.matcher(url.toExternalForm()); return m.matches(); @@ -74,7 +74,7 @@ public List getAlbumsToQueue(Document doc) { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { + public String getAlbumTitle() throws MalformedURLException, URISyntaxException { try { // Attempt to use album title as GID return getHost() + "_" + getGID(url) + "_" + getCachedFirstPage().select("title").first().text().replaceAll(" ", "_"); @@ -82,7 +82,7 @@ public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxExce // Fall back to default album naming convention logger.info("Unable to find title at " + url); } - return super.getAlbumTitle(url); + return super.getAlbumTitle(); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java index f2f5c9599..17234cd0c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java @@ -108,7 +108,7 @@ public String getHost() { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException { + public String getAlbumTitle() throws MalformedURLException { try { // Attempt to use album title as GID Document doc = getCachedFirstPage(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java index 8bd67c5f2..c1600cc69 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java @@ -56,7 +56,7 @@ public boolean canRip(URL url) { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { + public String getAlbumTitle() throws MalformedURLException, URISyntaxException { try { // Attempt to use album title as GID Element titleElement = getCachedFirstPage().select("meta[property=og:title]").first(); @@ -67,7 +67,7 @@ public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxExce // Fall back to default album naming convention logger.info("Unable to find title at " + url); } - return super.getAlbumTitle(url); + return super.getAlbumTitle(); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java index 5349f55c7..d0eef48ce 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java @@ -68,7 +68,8 @@ public String getDomain() { return "e-hentai.org"; } - public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { + @Override + public String getAlbumTitle() throws MalformedURLException, URISyntaxException { try { // Attempt to use album title as GID if (albumDoc == null) { @@ -80,7 +81,7 @@ public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxExce // Fall back to default album naming convention logger.warn("Failed to get album title from " + url, e); } - return super.getAlbumTitle(url); + return super.getAlbumTitle(); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java index 8765c4daf..da7492181 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java @@ -59,7 +59,7 @@ public String getGID(URL url) throws MalformedURLException { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { + public String getAlbumTitle() throws MalformedURLException, URISyntaxException { try { // Attempt to use album title as GID Element titleElement = getCachedFirstPage().select("meta[name=description]").first(); @@ -71,14 +71,12 @@ public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxExce // Fall back to default album naming convention logger.info("Unable to find title at " + url); } - return super.getAlbumTitle(url); + return super.getAlbumTitle(); } @Override public Document getFirstPage() throws IOException { - Response resp = Http.url(url).response(); - cookies.putAll(resp.cookies()); - return resp.parse(); + return Http.url(url).collectCookiesInto(cookies).get(); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java index 146c15f3e..ee5966962 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java @@ -57,7 +57,7 @@ public boolean hasQueueSupport() { } @Override - public boolean pageContainsAlbums(URL url) { + public boolean pageContainsAlbums() { Pattern pa = Pattern.compile("https?://www.erome.com/([a-zA-Z0-9_\\-?=]*)/?"); Matcher ma = pa.matcher(url.toExternalForm()); return ma.matches(); @@ -73,7 +73,7 @@ public List getAlbumsToQueue(Document doc) { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { + public String getAlbumTitle() throws MalformedURLException, URISyntaxException { try { // Attempt to use album title as GID Element titleElement = getCachedFirstPage().select("meta[property=og:title]").first(); @@ -86,7 +86,7 @@ public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxExce } catch (NullPointerException e) { return getHost() + "_" + getGID(url); } - return super.getAlbumTitle(url); + return super.getAlbumTitle(); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FapDungeonRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FapDungeonRipper.java index 20f3a778e..1929c0858 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FapDungeonRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FapDungeonRipper.java @@ -136,12 +136,12 @@ public void downloadURL(URL url, int index) { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { + public String getAlbumTitle() throws MalformedURLException, URISyntaxException { Matcher m = pagePattern.matcher(url.toExternalForm()); if (m.matches()) { return getHost() + "_" + m.group(1) + "_" + m.group(2); } else { - return super.getAlbumTitle(url); + return super.getAlbumTitle(); } } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java index 1f1954207..f7075c4fe 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java @@ -179,9 +179,9 @@ private Album getAlbum(String url) throws MalformedURLException { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { + public String getAlbumTitle() throws MalformedURLException, URISyntaxException { if (!url.toExternalForm().contains("/sets/")) { - return super.getAlbumTitle(url); + return super.getAlbumTitle(); } try { // Attempt to use album title as GID @@ -196,7 +196,7 @@ public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxExce } catch (Exception e) { // Fall back to default album naming convention } - return super.getAlbumTitle(url); + return super.getAlbumTitle(); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java index ad9075d0d..bd5c11f86 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java @@ -152,19 +152,18 @@ public int descSleepTime() { public String getDescription(String page) { try { // Fetch the image page - Response resp = Http.url(page) + Document documentz = Http.url(page) .referrer(this.url) - .response(); - cookies.putAll(resp.cookies()); + .collectCookiesInto(cookies) + .get(); // Try to find the description - Elements els = resp.parse().select("td[class=alt1][width=\"70%\"]"); + Elements els = documentz.select("td[class=alt1][width=\"70%\"]"); if (els.isEmpty()) { logger.debug("No description at " + page); throw new IOException("No description found"); } logger.debug("Description found!"); - Document documentz = resp.parse(); Element ele = documentz.select("td[class=alt1][width=\"70%\"]").get(0); // This is where the description is. // Would break completely if FurAffinity changed site layout. documentz.outputSettings(new Document.OutputSettings().prettyPrint(false)); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GirlsOfDesireRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GirlsOfDesireRipper.java index e6983febd..299d66efb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/GirlsOfDesireRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GirlsOfDesireRipper.java @@ -35,7 +35,8 @@ public String getDomain() { return "girlsofdesire.org"; } - public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { + @Override + public String getAlbumTitle() throws MalformedURLException, URISyntaxException { try { // Attempt to use album title as GID Document doc = getCachedFirstPage(); @@ -45,7 +46,7 @@ public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxExce // Fall back to default album naming convention logger.warn("Failed to get album title from " + url, e); } - return super.getAlbumTitle(url); + return super.getAlbumTitle(); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java index 91cf47d55..568ed7e7f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java @@ -43,7 +43,7 @@ public boolean hasQueueSupport() { } @Override - public boolean pageContainsAlbums(URL url) { + public boolean pageContainsAlbums() { logger.info("Page contains albums"); Pattern pat = Pattern.compile("https?://hentai2read\\.com/([a-zA-Z0-9_-]*)/?"); Matcher mat = pat.matcher(url.toExternalForm()); @@ -78,7 +78,7 @@ public Document getFirstPage() throws IOException { String thumbnailLink; try { // If the page contains albums we want to load the main page - if (pageContainsAlbums(url)) { + if (pageContainsAlbums()) { return Http.url(url).get(); } Document tempDoc; @@ -96,14 +96,14 @@ public Document getFirstPage() throws IOException { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { + public String getAlbumTitle() throws MalformedURLException, URISyntaxException { try { return getHost() + "_" + getGID(url); } catch (Exception e) { // Fall back to default album naming convention logger.warn("Failed to get album title from " + url, e); } - return super.getAlbumTitle(url); + return super.getAlbumTitle(); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java index 6184c7fc6..746556ff3 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java @@ -12,15 +12,13 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.jsoup.Connection.Method; -import org.jsoup.Connection.Response; +import com.rarchives.ripme.utils.Utils; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.utils.Http; -import com.rarchives.ripme.utils.Utils; public class HentaifoundryRipper extends AbstractHTMLRipper { @@ -55,17 +53,15 @@ public String getGID(URL url) throws MalformedURLException { @Override public Document getFirstPage() throws IOException { - Response resp; Document doc; - resp = Http.url("https://www.hentai-foundry.com/?enterAgree=1&size=1500") + doc = Http.url("https://www.hentai-foundry.com/?enterAgree=1&size=1500") .referrer("https://www.hentai-foundry.com/") .cookies(cookies) - .response(); - // The only cookie that seems to matter in getting around the age wall is the phpsession cookie - cookies.putAll(resp.cookies()); + // The only cookie that seems to matter in getting around the age wall is the phpsession cookie + .collectCookiesInto(cookies) + .get(); - doc = resp.parse(); String csrf_token = doc.select("input[name=YII_CSRF_TOKEN]") .first().attr("value"); if (csrf_token != null) { @@ -94,24 +90,22 @@ public Document getFirstPage() throws IOException { data.put("filter_order" , Utils.getConfigString("hentai-foundry.filter_order","date_old")); data.put("filter_type" , "0"); - resp = Http.url("https://www.hentai-foundry.com/site/filters") + doc = Http.url("https://www.hentai-foundry.com/site/filters") .referrer("https://www.hentai-foundry.com/") .cookies(cookies) + .collectCookiesInto(cookies) .data(data) - .method(Method.POST) - .response(); - cookies.putAll(resp.cookies()); + .post(); } else { logger.info("unable to find csrf_token and set filter"); } - resp = Http.url(url) + return Http.url(url) .referrer("https://www.hentai-foundry.com/") .cookies(cookies) - .response(); - cookies.putAll(resp.cookies()); - return resp.parse(); + .collectCookiesInto(cookies) + .get(); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoxRipper.java index 2858725b6..31a9df64e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoxRipper.java @@ -57,7 +57,7 @@ public List getURLsFromPage(Document doc) { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { + public String getAlbumTitle() throws MalformedURLException, URISyntaxException { try { Document doc = getCachedFirstPage(); String title = doc.select("div.info > h1").first().text(); @@ -66,7 +66,7 @@ public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxExce // Fall back to default album naming convention logger.warn("Failed to get album title from " + url, e); } - return super.getAlbumTitle(url); + return super.getAlbumTitle(); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java index 186fa7336..491743c6a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java @@ -70,7 +70,7 @@ public List getURLsFromPage(Document doc) { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { + public String getAlbumTitle() throws MalformedURLException, URISyntaxException { try { // Attempt to use album title and username as GID Document doc = Http.url(url).get(); @@ -79,7 +79,7 @@ public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxExce } catch (IOException e) { logger.info("Falling back"); } - return super.getAlbumTitle(url); + return super.getAlbumTitle(); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java index d3e5b2ce8..37c02af41 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java @@ -95,7 +95,7 @@ public void downloadURL(URL url, int index) { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { + public String getAlbumTitle() throws MalformedURLException, URISyntaxException { try { // Attempt to use album title as GID Elements elems = getCachedFirstPage().select("[id=gallery-name]"); @@ -108,7 +108,7 @@ public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxExce // Fall back to default album naming convention logger.warn("Failed to get album title from " + url, e); } - return super.getAlbumTitle(url); + return super.getAlbumTitle(); } /** diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java index 375ea06c9..64476a6ae 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java @@ -176,7 +176,7 @@ public void downloadURL(URL url, int index) { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { + public String getAlbumTitle() throws MalformedURLException, URISyntaxException { try { // Attempt to use album title as GID String title = getCachedFirstPage().title(); @@ -185,7 +185,7 @@ public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxExce String toReturn = getHost() + "_" + title + "_" + getGID(url); return toReturn.replaceAll("__", "_"); } catch (IOException e) { - return super.getAlbumTitle(url); + return super.getAlbumTitle(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java index 5bb7b0020..af879908e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java @@ -108,7 +108,7 @@ public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException } @Override - public String getAlbumTitle(URL url) throws MalformedURLException { + public String getAlbumTitle() throws MalformedURLException { String gid = null; try { gid = getGID(url); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java index deedfb888..1e8726b6a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java @@ -58,7 +58,7 @@ public boolean hasQueueSupport() { } @Override - public boolean pageContainsAlbums(URL url) { + public boolean pageContainsAlbums() { Pattern pa = Pattern.compile("^https?://myhentaicomics.com/index.php/search\\?q=([a-zA-Z0-9-]*)([a-zA-Z0-9=&]*)?$"); Matcher ma = pa.matcher(url.toExternalForm()); if (ma.matches()) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java index d6b17b02f..4dda06a83 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java @@ -152,7 +152,7 @@ public boolean hasQueueSupport() { } @Override - public boolean pageContainsAlbums(URL url) { + public boolean pageContainsAlbums() { try { final var fstPage = getCachedFirstPage(); List imageURLs = getImagePageURLs(fstPage); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java index 41693a3ee..51824793c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java @@ -2,6 +2,7 @@ import java.io.IOException; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -36,7 +37,7 @@ public boolean hasQueueSupport() { } @Override - public boolean pageContainsAlbums(URL url) { + public boolean pageContainsAlbums() { Pattern pa = Pattern.compile("^https?://nhentai\\.net/tag/([a-zA-Z0-9_\\-]+)/?"); Matcher ma = pa.matcher(url.toExternalForm()); return ma.matches(); @@ -71,7 +72,7 @@ public String getHost() { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException { + public String getAlbumTitle() throws MalformedURLException, URISyntaxException { if (firstPage == null) { try { firstPage = Http.url(url).get(); @@ -82,7 +83,7 @@ public String getAlbumTitle(URL url) throws MalformedURLException { String title = firstPage.select("#info > h1").text(); if (title == null) { - return getAlbumTitle(url); + return super.getAlbumTitle(); } return "nhentai" + title; } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/OglafRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/OglafRipper.java index e03d3bdcd..208255824 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/OglafRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/OglafRipper.java @@ -42,7 +42,7 @@ public String getGID(URL url) throws MalformedURLException { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException { + public String getAlbumTitle() throws MalformedURLException { return getDomain(); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index 2419cd035..921deba0d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -141,7 +141,6 @@ private URL getAndParseAndReturnNext(URL url) throws IOException, URISyntaxExcep children.getJSONObject(j).getJSONObject("data").getBoolean("is_self") ) { URL selfPostURL = new URI(children.getJSONObject(j).getJSONObject("data").getString("url")).toURL(); - System.out.println(selfPostURL.toExternalForm()); saveText(getJsonArrayFromURL(getJsonURL(selfPostURL))); } } catch (Exception e) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SankakuComplexRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SankakuComplexRipper.java index 7e6899da1..acbed239f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/SankakuComplexRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/SankakuComplexRipper.java @@ -14,7 +14,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.jsoup.Connection.Response; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -25,7 +24,6 @@ public class SankakuComplexRipper extends AbstractHTMLRipper { private static final Logger logger = LogManager.getLogger(SankakuComplexRipper.class); - private Document albumDoc = null; private Map cookies = new HashMap<>(); public SankakuComplexRipper(URL url) throws IOException { @@ -74,12 +72,7 @@ public String getSubDomain(URL url){ @Override public Document getFirstPage() throws IOException { - if (albumDoc == null) { - Response resp = Http.url(url).response(); - cookies.putAll(resp.cookies()); - albumDoc = resp.parse(); - } - return albumDoc; + return Http.url(url).collectCookiesInto(cookies).get(); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/StaRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/StaRipper.java index d6faae9fc..306ac9fd0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/StaRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/StaRipper.java @@ -61,9 +61,7 @@ public List getURLsFromPage(Document doc) { Document thumbPage = null; if (checkURL(thumbPageURL)) { try { - Connection.Response resp = Http.url(new URI(thumbPageURL).toURL()).response(); - cookies.putAll(resp.cookies()); - thumbPage = resp.parse(); + thumbPage = Http.url(new URI(thumbPageURL).toURL()).collectCookiesInto(cookies).get(); } catch (MalformedURLException | URISyntaxException e) { logger.info(thumbPageURL + " is a malformed URL"); } catch (IOException e) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java index 2b003ab09..0eeb6389d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java @@ -103,9 +103,7 @@ private String getAlbumID() { @Override public Document getFirstPage() throws IOException { - Connection.Response resp = Http.url(url).response(); - cookies.putAll(resp.cookies()); - Document doc = resp.parse(); + Document doc = Http.url(url).collectCookiesInto(cookies).get(); String blacklistedTag = RipUtils.checkTags(Utils.getConfigStringArray("tsumino.blacklist.tags"), getTags(doc)); if (blacklistedTag != null) { sendUpdate(RipStatusMessage.STATUS.DOWNLOAD_WARN, "Skipping " + url.toExternalForm() + " as it " + diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ViewcomicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ViewcomicRipper.java index 6ab9e7449..74ba45ea3 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ViewcomicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ViewcomicRipper.java @@ -35,7 +35,7 @@ public String getDomain() { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { + public String getAlbumTitle() throws MalformedURLException, URISyntaxException { try { // Attempt to use album title as GID String titleText = getCachedFirstPage().select("title").first().text(); @@ -49,7 +49,7 @@ public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxExce // Fall back to default album naming convention logger.info("Unable to find title at " + url); } - return super.getAlbumTitle(url); + return super.getAlbumTitle(); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java index 0da345b7e..5da5921d0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java @@ -44,14 +44,14 @@ public boolean canRip(URL url) { @Override - public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { + public String getAlbumTitle() throws MalformedURLException, URISyntaxException { Pattern pat = Pattern.compile("https?://www.webtoons.com/[a-zA-Z-_]+/[a-zA-Z_-]+/([a-zA-Z0-9_-]*)/[a-zA-Z0-9_-]+/\\S*"); Matcher mat = pat.matcher(url.toExternalForm()); if (mat.matches()) { return getHost() + "_" + mat.group(1); } - return super.getAlbumTitle(url); + return super.getAlbumTitle(); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java index 6c6962721..cb43733f0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java @@ -189,7 +189,7 @@ public boolean hasQueueSupport() { } @Override - public boolean pageContainsAlbums(URL url) { + public boolean pageContainsAlbums() { Pattern pa = Pattern.compile("^https?://8muses.download/\\?s=([a-zA-Z0-9-]*)"); Matcher ma = pa.matcher(url.toExternalForm()); if (ma.matches()) { @@ -221,7 +221,7 @@ public List getAlbumsToQueue(Document doc) { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { + public String getAlbumTitle() throws MalformedURLException, URISyntaxException { Pattern totempole666Pat = Pattern.compile("(?:https?://)?(?:www\\.)?totempole666.com/comic/([a-zA-Z0-9_-]*)/?$"); Matcher totempole666Mat = totempole666Pat.matcher(url.toExternalForm()); if (totempole666Mat.matches()) { @@ -300,7 +300,7 @@ public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxExce return "comixfap_" + mat.group(1); } - return super.getAlbumTitle(url); + return super.getAlbumTitle(); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java index 4903d22fc..23e85201c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java @@ -103,7 +103,7 @@ public boolean hasQueueSupport() { } @Override - public boolean pageContainsAlbums(URL url) { + public boolean pageContainsAlbums() { Pattern p = Pattern.compile("^https?://[\\w\\w.]*xhamster([^<]*)\\.(com|desi)/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?"); Matcher m = p.matcher(url.toExternalForm()); logger.info("Checking if page has albums"); @@ -220,7 +220,7 @@ private void downloadFile(String url) { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { + public String getAlbumTitle() throws MalformedURLException, URISyntaxException { try { // Attempt to use album title and username as GID Document doc = getCachedFirstPage(); @@ -235,6 +235,6 @@ public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxExce } catch (IOException | NullPointerException e) { // Fall back to default album naming convention } - return super.getAlbumTitle(url); + return super.getAlbumTitle(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java index 119e0306d..337ee8192 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java @@ -110,7 +110,7 @@ public void downloadURL(URL url, int index) { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { + public String getAlbumTitle() throws MalformedURLException, URISyntaxException { Pattern p; Matcher m; @@ -126,6 +126,6 @@ public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxExce return getHost() + "_" + m.group(1) + "_" + m.group(2) + "_" + m.group(4) + "_" + m.group(3); } - return super.getAlbumTitle(url); + return super.getAlbumTitle(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java index 1240f33c5..d379ba725 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java @@ -13,7 +13,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.jsoup.Connection.Response; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -50,7 +49,7 @@ public String getGID(URL url) throws MalformedURLException { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { + public String getAlbumTitle() throws MalformedURLException, URISyntaxException { try { // Attempt to use album title as GID Element titleElement = getCachedFirstPage().select("h1.title").first(); @@ -64,14 +63,12 @@ public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxExce // Fall back to default album naming convention logger.info("Unable to find title at " + url); } - return super.getAlbumTitle(url); + return super.getAlbumTitle(); } @Override public Document getFirstPage() throws IOException { - Response resp = Http.url(url).response(); - cookies.putAll(resp.cookies()); - return resp.parse(); + return Http.url(url).collectCookiesInto(cookies).get(); } @Override diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 35357830c..4e4ab9918 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -1543,6 +1543,8 @@ private synchronized void handleEvent(StatusEvent evt) { } break; + case DOWNLOAD_PROGRESSED: + break; case RIP_ERRORED: if (LOGGER.isEnabled(Level.ERROR)) { appendLog((String) msg.getObject(), Color.RED); @@ -1569,7 +1571,7 @@ private synchronized void handleEvent(StatusEvent evt) { entry.dir = rsc.getDir(); entry.count = rsc.count; try { - entry.title = ripper.getAlbumTitle(ripper.getURL()); + entry.title = ripper.getAlbumTitle(); } catch (MalformedURLException | URISyntaxException e) { LOGGER.warn(e.getMessage()); } diff --git a/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java b/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java index f589e9dbb..9a5032e3a 100644 --- a/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java +++ b/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java @@ -10,6 +10,7 @@ public enum STATUS { DOWNLOAD_STARTED("Download Started"), DOWNLOAD_COMPLETE("Download Complete"), DOWNLOAD_ERRORED("Download Errored"), + DOWNLOAD_PROGRESSED("Download Progressed"), DOWNLOAD_COMPLETE_HISTORY("Download Complete History"), RIP_COMPLETE("Rip Complete"), DOWNLOAD_WARN("Download problem"), diff --git a/src/main/java/com/rarchives/ripme/utils/Http.java b/src/main/java/com/rarchives/ripme/utils/Http.java index a1705f5a9..67f166afa 100644 --- a/src/main/java/com/rarchives/ripme/utils/Http.java +++ b/src/main/java/com/rarchives/ripme/utils/Http.java @@ -38,6 +38,7 @@ public class Http { private int retrySleep = 0; private final String url; private Connection connection; + private Map collectedCookies; // Constructors public Http(String url) { @@ -58,6 +59,11 @@ public static Http url(URL url) { return new Http(url); } + public Http collectCookiesInto(Map collectedCookies) { + this.collectedCookies = collectedCookies; + return this; + } + private void defaultSettings() { this.retries = Utils.getConfigInteger("download.retries", 3); this.retrySleep = Utils.getConfigInteger("download.retry.sleep", 5000); @@ -203,6 +209,9 @@ public Response response() throws IOException { while (--retries >= 0) { try { response = connection.execute(); + if (collectedCookies != null) { + collectedCookies.putAll(response.cookies()); + } return response; } catch (IOException e) { // Warn users about possibly fixable permission error diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java index 4fee248da..eb39ecdda 100644 --- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java +++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java @@ -6,6 +6,7 @@ import java.net.URISyntaxException; import java.net.URL; import java.util.*; +import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -21,6 +22,7 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; public class RipUtils { @@ -78,11 +80,7 @@ public static List getFilesFromURL(URL url) { } else if (url.toExternalForm().contains("erome.com")) { try { logger.info("Getting eroshare album " + url); - EromeRipper r = new EromeRipper(url); - Document tempDoc = r.getFirstPage(); - for (String u : r.getURLsFromPage(tempDoc)) { - result.add(new URI(u).toURL()); - } + result.addAll(new EromeRipper(url).getURLsFromFirstPage()); } catch (IOException | URISyntaxException e) { // Do nothing logger.warn("Exception while retrieving eroshare page:", e); @@ -91,11 +89,7 @@ public static List getFilesFromURL(URL url) { } else if (url.toExternalForm().contains("soundgasm.net")) { try { logger.info("Getting soundgasm page " + url); - SoundgasmRipper r = new SoundgasmRipper(url); - Document tempDoc = r.getFirstPage(); - for (String u : r.getURLsFromPage(tempDoc)) { - result.add(new URI(u).toURL()); - } + result.addAll(new SoundgasmRipper(url).getURLsFromFirstPage()); } catch (IOException | URISyntaxException e) { // Do nothing logger.warn("Exception while retrieving soundgasm page:", e); @@ -317,4 +311,75 @@ public static String checkTags(String[] blackListedTags, List tagsOnPage } return null; } + + /** + * Create a new URL from the string, if it's a correctly formatted absolute url, returns null otherwise. + * @param href + * @return + */ + public static URL createFromAbsoluteUrl(final String href) { + if (href != null && !href.isBlank()) { + try { + return new URI(href).toURL(); + } catch (final MalformedURLException|URISyntaxException ex) { + logger.error("Malformed URL: " + href, ex); + } + } + return null; + } + + public static void addUrl(List urls, String absoluteUrl) { + final var url = createFromAbsoluteUrl(absoluteUrl); + if (url != null) { + urls.add(url); + } + } + + public static List toURLList(List urlStrings) { + final var correctUrls = new ArrayList(); + for (var url: urlStrings) { + try { + correctUrls.add(new URI(url).toURL()); + } catch (MalformedURLException | URISyntaxException ex) { + logger.error("Malformed URL: " + url, ex); + } + } + return correctUrls; + } + + public static List extractUrl(final Elements elements, String attributeName) { + return extractUrl(elements, attributeName, Function.identity()); + } + + public static List extractUrl(final Elements elements, String attributeName, Function urlBuilder) { + final var result = new ArrayList(); + for (final var elem : elements) { + final var href = elem.attr(attributeName); + final var absoluteUrl = urlBuilder.apply(href); + final var url = createFromAbsoluteUrl(absoluteUrl); + if (url != null) { + result.add(url); + } + } + return result; + } + + public static List extractUrlAsString(final Elements elements, String attributeName, Function urlBuilder) { + return extractUrl(elements, attributeName, urlBuilder).stream().map(URL::toExternalForm).toList(); + } + + public static String cutEverythingAfter(String string, String start) { + var pos = string.indexOf(start); + if (pos >= 0) { + return string.substring(0, pos); + } + return string; + } + + public static String removeIfStartsWith(String string, String prefix) { + if (string.startsWith(prefix)) { + return string.substring(prefix.length()); + } + return null; + } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BatoRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BatoRipperTest.java index 6849f0e17..f86922901 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BatoRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BatoRipperTest.java @@ -32,6 +32,6 @@ public void testGetGID() throws IOException, URISyntaxException { public void testGetAlbumTitle() throws IOException, URISyntaxException { URL url = new URI("https://bato.to/chapter/1207152").toURL(); BatoRipper ripper = new BatoRipper(url); - Assertions.assertEquals("bato_1207152_I_Messed_Up_by_Teaching_at_a_Black_Gyaru_School!_Ch.2", ripper.getAlbumTitle(url)); + Assertions.assertEquals("bato_1207152_I_Messed_Up_by_Teaching_at_a_Black_Gyaru_School!_Ch.2", ripper.getAlbumTitle()); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EromeRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EromeRipperTest.java index 392a245af..e27ce9d59 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EromeRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EromeRipperTest.java @@ -60,7 +60,7 @@ public void testGetAlbumsToQueue() throws IOException, URISyntaxException { public void testPageContainsAlbums() throws IOException, URISyntaxException { URL url = new URI(USER_PAGE_JAY_JENNA).toURL(); EromeRipper ripper = new EromeRipper(url); - assert (ripper.pageContainsAlbums(url)); + assert (ripper.pageContainsAlbums()); } @Test diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java index 2af7d4999..196401a6c 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java @@ -32,6 +32,6 @@ public void testImagefapAlbums() throws IOException, URISyntaxException { public void testImagefapGetAlbumTitle() throws IOException, URISyntaxException { URL url = new URI("https://www.imagefap.com/pictures/11365460/Cartoons").toURL(); ImagefapRipper ripper = new ImagefapRipper(url); - Assertions.assertEquals("imagefap_Cartoons_11365460", ripper.getAlbumTitle(url)); + Assertions.assertEquals("imagefap_Cartoons_11365460", ripper.getAlbumTitle()); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java index 52f8b0d85..ce9795e5e 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java @@ -40,7 +40,7 @@ public void testPageContainsAlbums() throws IOException, URISyntaxException { URL url = new URI("https://myhentaicomics.com/index.php/tag/3167/").toURL(); URL url2 = new URI("https://myhentaicomics.com/index.php/search?q=test").toURL(); MyhentaicomicsRipper ripper = new MyhentaicomicsRipper(url); - Assertions.assertTrue(ripper.pageContainsAlbums(url)); - Assertions.assertTrue(ripper.pageContainsAlbums(url2)); + Assertions.assertTrue(ripper.pageContainsAlbums()); + Assertions.assertTrue(new MyhentaicomicsRipper(url2).pageContainsAlbums()); } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java index d0649aa93..b9fec792c 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java @@ -71,7 +71,7 @@ public void test_konradokonski_2() throws IOException, URISyntaxException { public void test_konradokonski_getAlbumTitle() throws IOException, URISyntaxException { URL url = new URI("http://www.konradokonski.com/sawdust/comic/get-up/").toURL(); WordpressComicRipper ripper = new WordpressComicRipper(url); - Assertions.assertEquals("konradokonski.com_sawdust", ripper.getAlbumTitle(url)); + Assertions.assertEquals("konradokonski.com_sawdust", ripper.getAlbumTitle()); } @@ -99,7 +99,7 @@ public void test_Eightmuses_download() throws IOException, URISyntaxException { public void test_Eightmuses_getAlbumTitle() throws IOException, URISyntaxException { URL url = new URI("https://8muses.download/lustomic-playkittens-josh-samuel-porn-comics-8-muses/").toURL(); WordpressComicRipper ripper = new WordpressComicRipper(url); - Assertions.assertEquals("8muses.download_lustomic-playkittens-josh-samuel-porn-comics-8-muses", ripper.getAlbumTitle(url)); + Assertions.assertEquals("8muses.download_lustomic-playkittens-josh-samuel-porn-comics-8-muses", ripper.getAlbumTitle()); } @Test @Tag("flaky") @@ -112,7 +112,7 @@ public void test_spyingwithlana_download() throws IOException, URISyntaxExceptio public void test_spyingwithlana_getAlbumTitle() throws IOException, URISyntaxException { URL url = new URI("http://spyingwithlana.com/comic/the-big-hookup/").toURL(); WordpressComicRipper ripper = new WordpressComicRipper(url); - Assertions.assertEquals("spyingwithlana_the-big-hookup", ripper.getAlbumTitle(url)); + Assertions.assertEquals("spyingwithlana_the-big-hookup", ripper.getAlbumTitle()); } @Test diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ZizkiRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ZizkiRipperTest.java index adbd4c77c..575284ac3 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ZizkiRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ZizkiRipperTest.java @@ -31,6 +31,6 @@ public void testGetGID() throws IOException, URISyntaxException { public void testAlbumTitle() throws IOException, URISyntaxException { URL url = new URI("http://zizki.com/dee-chorde/we-got-spirit").toURL(); ZizkiRipper ripper = new ZizkiRipper(url); - Assertions.assertEquals("zizki_Dee Chorde_We Got Spirit", ripper.getAlbumTitle(url)); + Assertions.assertEquals("zizki_Dee Chorde_We Got Spirit", ripper.getAlbumTitle()); } }