Skip to content

Pr/1601 #1625

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
swlovely50 wants to merge 2 commits into from
+42 −31
Closed

Pr/1601 #1625

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ef4ad7b
Update XhamsterRipper.java
ehump12000 Mar 13, 2020
fa9c5b8
Merge remote-tracking branch 'upstream/master'
ehump12000 Mar 30, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 42 additions & 31 deletions src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,19 @@
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.Connection.Response;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;

import com.rarchives.ripme.ripper.AbstractHTMLRipper;
import com.rarchives.ripme.utils.Http;


// WARNING
// This ripper changes all requests to use the MOBILE version of the site
// If you're changing anything be sure to use the mobile sites html/css or you're just wasting your time!
Expand All @@ -27,8 +29,11 @@ public XhamsterRipper(URL url) throws IOException {
}

private int index = 1;
private Document albumDoc = null;
private Map<String, String> cookies = new HashMap<>();

@Override public boolean hasASAPRipping() {
@Override
public boolean hasASAPRipping() {
return true;
}

Expand Down Expand Up @@ -73,10 +78,8 @@ public String getGID(URL url) throws MalformedURLException {
return m.group(2);
}

throw new MalformedURLException(
"Expected xhamster.com gallery formats: "
+ "xhamster.com/photos/gallery/xxxxx-#####"
+ " Got: " + url);
throw new MalformedURLException(
"Expected xhamster.com gallery formats: " + "xhamster.com/photos/gallery/xxxxx-#####" + " Got: " + url);
}

@Override
Expand All @@ -97,18 +100,22 @@ public boolean hasQueueSupport() {

@Override
public boolean pageContainsAlbums(URL url) {
Pattern p = Pattern.compile("^https?://[\\w\\w.]*xhamster\\.com/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?");
Pattern p = Pattern
.compile("^https?://[\\w\\w.]*xhamster\\.com/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?");
Matcher m = p.matcher(url.toExternalForm());
LOGGER.info("Checking if page has albums");
LOGGER.info(m.matches());
return m.matches();
}


@Override
public Document getFirstPage() throws IOException {
// "url" is an instance field of the superclass
return Http.url(url).get();
if (albumDoc == null) {
Response resp = Http.url(url).response();
cookies.putAll(resp.cookies());
albumDoc = resp.parse();
}
return albumDoc;
}

@Override
Expand All @@ -118,7 +125,8 @@ public boolean canRip(URL url) {
if (m.matches()) {
return true;
}
p = Pattern.compile("^https?://[\\w\\w.]*xhamster2?\\.(com|one|desi)/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?");
p = Pattern.compile(
"^https?://[\\w\\w.]*xhamster2?\\.(com|one|desi)/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?");
m = p.matcher(url.toExternalForm());
if (m.matches()) {
return true;
Expand Down Expand Up @@ -156,23 +164,25 @@ public List<String> getURLsFromPage(Document doc) {
LOGGER.debug("Checking for urls");
List<String> result = new ArrayList<>();
if (!isVideoUrl(url)) {
for (Element page : doc.select("div.picture_view > div.pictures_block > div.items > div.item-container > a.item")) {
// Make sure we don't waste time running the loop if the ripper has been stopped
if (isStopped()) {
break;
}
String pageWithImageUrl = page.attr("href");
try {
// This works around some redirect fuckery xhamster likes to do where visiting m.xhamster.com sends to
// the page chamster.com but displays the mobile site from m.xhamster.com
pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster\\.", "://m.xhamster.");
pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster2\\.", "://m.xhamster.");
String image = Http.url(new URL(pageWithImageUrl)).get().select("a > img#photoCurr").attr("src");
downloadFile(image);
} catch (IOException e) {
LOGGER.error("Was unable to load page " + pageWithImageUrl);
}
}
for (Element page : doc
.select("div.picture_view > div.pictures_block > div.items > div.item-container > a.item")) {
// Make sure we don't waste time running the loop if the ripper has been stopped
if (isStopped()) {
break;
}
String pageWithImageUrl = page.attr("href");
try {
// This works around some redirect fuckery xhamster likes to do where visiting
// m.xhamster.com sends to
// the page chamster.com but displays the mobile site from m.xhamster.com
pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster\\.", "://m.xhamster.");
pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster2\\.", "://m.xhamster.");
String image = Http.url(new URL(pageWithImageUrl)).get().select("a > img#photoCurr").attr("src");
downloadFile(image);
} catch (IOException e) {
LOGGER.error("Was unable to load page " + pageWithImageUrl);
}
}
} else {
String imgUrl = doc.select("div.player-container > a").attr("href");
downloadFile(imgUrl);
Expand All @@ -182,18 +192,19 @@ public List<String> getURLsFromPage(Document doc) {

@Override
public void downloadURL(URL url, int index) {
addURLToDownload(url, getPrefix(index));
// addURLToDownload(url, getPrefix(index));
addURLToDownload(url, getPrefix(index), "", this.url.toExternalForm(), cookies);
}

private void downloadFile(String url) {
try {
addURLToDownload(new URL(url), getPrefix(index));
addURLToDownload(new URL(url), getPrefix(index), "", this.url.toExternalForm(), cookies);
index = index + 1;
} catch (MalformedURLException e) {
LOGGER.error("The url \"" + url + "\" is malformed");
}
}

@Override
public String getAlbumTitle(URL url) throws MalformedURLException {
try {
Expand Down