From f39d3100c46474d56f58150ef0c090302f01bea4 Mon Sep 17 00:00:00 2001 From: Rogiel Date: Tue, 13 Sep 2011 23:40:39 -0300 Subject: [PATCH] Add support for Eclipse 3.7 and implement HTMLParser in services This commit adds support for Eclipse Indigo (3.7), with native Maven2Eclipse (m2e). It also adds an HTML parser to be used while uploading and downloading. --- .classpath | 4 +- .project | 3 +- .settings/org.eclipse.jdt.core.prefs | 8 +- .settings/org.eclipse.m2e.core.prefs | 5 + pom.xml | 28 +- .../service/AbstractHttpService.java | 106 ++++++++ .../httpchannel/service/ServiceHelper.java | 50 ++++ .../LinkedUploadChannelContentBody.java} | 8 +- .../service/impl/HotFileService.java | 78 ++---- .../service/impl/MegaUploadService.java | 76 +++--- .../rogiel/httpchannel/util/ChannelUtils.java | 51 ++++ .../httpchannel/util/HttpClientUtils.java | 6 + .../httpchannel/util/htmlparser/HTMLPage.java | 245 ++++++++++++++++++ .../service/impl/HotFileServiceTest.java | 77 +++--- .../service/impl/MegaUploadServiceTest.java | 71 +++-- src/test/resources/upload-test-file.txt | 3 + 16 files changed, 624 insertions(+), 195 deletions(-) create mode 100644 .settings/org.eclipse.m2e.core.prefs create mode 100644 src/main/java/com/rogiel/httpchannel/service/ServiceHelper.java rename src/main/java/com/rogiel/httpchannel/service/{UploadListenerContentBody.java => channel/LinkedUploadChannelContentBody.java} (88%) create mode 100644 src/main/java/com/rogiel/httpchannel/util/ChannelUtils.java create mode 100644 src/main/java/com/rogiel/httpchannel/util/htmlparser/HTMLPage.java create mode 100644 src/test/resources/upload-test-file.txt diff --git a/.classpath b/.classpath index edcdd6b..6e26406 100644 --- a/.classpath +++ b/.classpath @@ -4,7 +4,7 @@ - - + + diff --git a/.project b/.project index c76c53e..84ceca0 100644 --- a/.project +++ b/.project @@ -11,12 +11,13 @@ - org.maven.ide.eclipse.maven2Builder + org.eclipse.m2e.core.maven2Builder + org.eclipse.m2e.core.maven2Nature org.eclipse.jdt.core.javanature org.maven.ide.eclipse.maven2Nature diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs index 2213c3f..f58b826 100644 --- a/.settings/org.eclipse.jdt.core.prefs +++ b/.settings/org.eclipse.jdt.core.prefs @@ -1,13 +1,13 @@ -#Sun Aug 14 13:47:37 BRT 2011 +#Sun Sep 11 18:22:22 BRT 2011 eclipse.preferences.version=1 org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=1.6 +org.eclipse.jdt.core.compiler.compliance=1.7 org.eclipse.jdt.core.compiler.debug.lineNumber=generate org.eclipse.jdt.core.compiler.debug.localVariable=generate org.eclipse.jdt.core.compiler.debug.sourceFile=generate org.eclipse.jdt.core.compiler.problem.assertIdentifier=error org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning -org.eclipse.jdt.core.compiler.source=1.6 +org.eclipse.jdt.core.compiler.source=1.7 diff --git a/.settings/org.eclipse.m2e.core.prefs b/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 0000000..b5b55f3 --- /dev/null +++ b/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,5 @@ +#Tue Sep 13 15:42:20 BRT 2011 +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/pom.xml b/pom.xml index 27a839d..42cc066 100644 --- a/pom.xml +++ b/pom.xml @@ -6,6 +6,20 @@ 1.0.0 Seedbox - HTTP Channel library Library capable of downloading and uploading files from free servers using channels. + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.7 + 1.7 + + + + + junit @@ -21,7 +35,10 @@ jar compile - commons-loggingcommons-logging + + commons-logging + commons-logging + @@ -52,5 +69,14 @@ jar compile + + org.htmlparser + htmlparser + 2.1 + jar + compile + + + \ No newline at end of file diff --git a/src/main/java/com/rogiel/httpchannel/service/AbstractHttpService.java b/src/main/java/com/rogiel/httpchannel/service/AbstractHttpService.java index 4b075f0..b0a3248 100644 --- a/src/main/java/com/rogiel/httpchannel/service/AbstractHttpService.java +++ b/src/main/java/com/rogiel/httpchannel/service/AbstractHttpService.java @@ -16,12 +16,25 @@ */ package com.rogiel.httpchannel.service; +import java.io.IOException; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.client.ClientProtocolException; import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; import org.apache.http.impl.client.DefaultHttpClient; import com.rogiel.httpchannel.service.captcha.CaptchaResolver; import com.rogiel.httpchannel.service.config.ServiceConfiguration; import com.rogiel.httpchannel.util.AlwaysRedirectStrategy; +import com.rogiel.httpchannel.util.HttpClientUtils; +import com.rogiel.httpchannel.util.htmlparser.HTMLPage; /** * Abstract base service for HTTP enabled services. @@ -31,6 +44,9 @@ import com.rogiel.httpchannel.util.AlwaysRedirectStrategy; */ public abstract class AbstractHttpService extends AbstractService implements Service { + private static final ExecutorService threadPool = Executors + .newCachedThreadPool(); + /** * The {@link HttpClient} instance for this service */ @@ -49,4 +65,94 @@ public abstract class AbstractHttpService // client.getParams().setIntParameter(ClientPNames.MAX_REDIRECTS, 10); // client.setRedirectStrategy(new DefaultRedirectStrategy()); } + + protected HttpResponse get(String url) throws ClientProtocolException, + IOException { + final HttpGet request = new HttpGet(url); + return client.execute(request); + } + + protected String getAsString(String url) throws ClientProtocolException, + IOException { + return HttpClientUtils.toString(get(url)); + } + + protected HTMLPage getAsPage(String url) throws ClientProtocolException, + IOException { + return HTMLPage.parse(getAsString(url)); + } + + public Future getAsync(final String url) throws IOException { + return threadPool.submit(new Callable() { + @Override + public HttpResponse call() throws Exception { + return get(url); + } + }); + } + + public Future getAsStringAsync(final String url) throws IOException { + return threadPool.submit(new Callable() { + @Override + public String call() throws Exception { + return getAsString(url); + } + }); + } + + public Future getAsPageAsync(final String url) throws IOException { + return threadPool.submit(new Callable() { + @Override + public HTMLPage call() throws Exception { + return getAsPage(url); + } + }); + } + + protected HttpResponse post(String url, HttpEntity entity) + throws ClientProtocolException, IOException { + final HttpPost request = new HttpPost(url); + request.setEntity(entity); + return client.execute(request); + } + + protected String postAsString(String url, HttpEntity entity) + throws ClientProtocolException, IOException { + return HttpClientUtils.toString(post(url, entity)); + } + + protected HTMLPage postAsPage(String url, HttpEntity entity) + throws ClientProtocolException, IOException { + return HTMLPage.parse(postAsString(url, entity)); + } + + protected Future postAsync(final String url, + final HttpEntity entity) throws IOException { + return threadPool.submit(new Callable() { + @Override + public HttpResponse call() throws Exception { + return post(url, entity); + } + }); + } + + protected Future postAsStringAsync(final String url, + final HttpEntity entity) throws IOException { + return threadPool.submit(new Callable() { + @Override + public String call() throws Exception { + return postAsString(url, entity); + } + }); + } + + protected Future postAsPageAsync(final String url, + final HttpEntity entity) throws IOException { + return threadPool.submit(new Callable() { + @Override + public HTMLPage call() throws Exception { + return postAsPage(url, entity); + } + }); + } } diff --git a/src/main/java/com/rogiel/httpchannel/service/ServiceHelper.java b/src/main/java/com/rogiel/httpchannel/service/ServiceHelper.java new file mode 100644 index 0000000..b2265f7 --- /dev/null +++ b/src/main/java/com/rogiel/httpchannel/service/ServiceHelper.java @@ -0,0 +1,50 @@ +/* + * This file is part of seedbox . + * + * seedbox is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * seedbox is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with seedbox. If not, see . + */ +package com.rogiel.httpchannel.service; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * @author Rogiel + */ +public class ServiceHelper { + private final Service service; + + /** + * @param service + * the service + */ + public ServiceHelper(Service service) { + this.service = service; + } + + public UploadChannel upload(Path path, String description) + throws IOException { + return ((UploadService) service).getUploader( + path.getFileName().toString(), Files.size(path), description) + .upload(); + } + + public UploadChannel upload(File file, String description) + throws IOException { + return ((UploadService) service).getUploader(file.getName(), + file.length(), description).upload(); + } +} diff --git a/src/main/java/com/rogiel/httpchannel/service/UploadListenerContentBody.java b/src/main/java/com/rogiel/httpchannel/service/channel/LinkedUploadChannelContentBody.java similarity index 88% rename from src/main/java/com/rogiel/httpchannel/service/UploadListenerContentBody.java rename to src/main/java/com/rogiel/httpchannel/service/channel/LinkedUploadChannelContentBody.java index 2bbdc18..a4a797f 100644 --- a/src/main/java/com/rogiel/httpchannel/service/UploadListenerContentBody.java +++ b/src/main/java/com/rogiel/httpchannel/service/channel/LinkedUploadChannelContentBody.java @@ -14,7 +14,7 @@ * You should have received a copy of the GNU General Public License * along with seedbox. If not, see . */ -package com.rogiel.httpchannel.service; +package com.rogiel.httpchannel.service.channel; import java.io.IOException; import java.io.OutputStream; @@ -25,7 +25,7 @@ import java.nio.channels.WritableByteChannel; import org.apache.http.entity.mime.content.AbstractContentBody; import org.apache.http.entity.mime.content.ContentBody; -import com.rogiel.httpchannel.service.channel.LinkedUploadChannel; +import com.rogiel.httpchannel.service.Uploader; import com.rogiel.httpchannel.util.ThreadUtils; /** @@ -34,10 +34,10 @@ import com.rogiel.httpchannel.util.ThreadUtils; * @author Rogiel * @since 1.0 */ -public class UploadListenerContentBody extends AbstractContentBody { +public class LinkedUploadChannelContentBody extends AbstractContentBody { private final LinkedUploadChannel channel; - public UploadListenerContentBody(LinkedUploadChannel channel) { + public LinkedUploadChannelContentBody(LinkedUploadChannel channel) { super("application/octet-stream"); this.channel = channel; } diff --git a/src/main/java/com/rogiel/httpchannel/service/impl/HotFileService.java b/src/main/java/com/rogiel/httpchannel/service/impl/HotFileService.java index 6ff60e8..42e656e 100644 --- a/src/main/java/com/rogiel/httpchannel/service/impl/HotFileService.java +++ b/src/main/java/com/rogiel/httpchannel/service/impl/HotFileService.java @@ -23,13 +23,11 @@ import java.util.concurrent.Future; import java.util.regex.Pattern; import org.apache.commons.io.FilenameUtils; -import org.apache.commons.io.IOUtils; import org.apache.http.HttpResponse; import org.apache.http.client.ClientProtocolException; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; import org.apache.http.entity.mime.MultipartEntity; import org.apache.http.entity.mime.content.StringBody; +import org.htmlparser.Tag; import com.rogiel.httpchannel.service.AbstractDownloader; import com.rogiel.httpchannel.service.AbstractHttpService; @@ -45,19 +43,18 @@ import com.rogiel.httpchannel.service.Downloader; import com.rogiel.httpchannel.service.DownloaderCapability; import com.rogiel.httpchannel.service.Service; import com.rogiel.httpchannel.service.UploadChannel; -import com.rogiel.httpchannel.service.UploadListenerContentBody; import com.rogiel.httpchannel.service.UploadService; import com.rogiel.httpchannel.service.Uploader; import com.rogiel.httpchannel.service.UploaderCapability; import com.rogiel.httpchannel.service.channel.InputStreamDownloadChannel; import com.rogiel.httpchannel.service.channel.LinkedUploadChannel; +import com.rogiel.httpchannel.service.channel.LinkedUploadChannelContentBody; import com.rogiel.httpchannel.service.channel.LinkedUploadChannel.LinkedUploadChannelCloseCallback; import com.rogiel.httpchannel.service.config.ServiceConfiguration; import com.rogiel.httpchannel.service.exception.AuthenticationInvalidCredentialException; import com.rogiel.httpchannel.service.impl.HotFileService.HotFileServiceConfiguration; -import com.rogiel.httpchannel.util.HttpClientUtils; -import com.rogiel.httpchannel.util.PatternUtils; import com.rogiel.httpchannel.util.ThreadUtils; +import com.rogiel.httpchannel.util.htmlparser.HTMLPage; /** * This service handles login, upload and download to HotFile.com. @@ -72,29 +69,14 @@ public class HotFileService extends .compile("http://u[0-9]*\\.hotfile\\.com/upload\\.cgi\\?[0-9]*"); private static final Pattern DOWNLOAD_DIRECT_LINK_PATTERN = Pattern - .compile("http://hotfile\\.com/get/([0-9]*)/([A-Za-z0-9]*)/([A-Za-z0-9]*)/([^\"]*)"); + .compile("http://hotfile\\.com/get/([0-9]*)/([A-Za-z0-9]*)/([A-Za-z0-9]*)/(.*)"); // private static final Pattern DOWNLOAD_TIMER = Pattern // .compile("timerend=d\\.getTime\\(\\)\\+([0-9]*);"); // private static final Pattern DOWNLOAD_FILESIZE = Pattern // .compile("[0-9]*(\\.[0-9]*)? (K|M|G)B"); private static final Pattern DOWNLOAD_URL_PATTERN = Pattern - .compile("http://hotfile\\.com/dl/([0-9]*)/([A-Za-z0-9]*)/([^\"]*)"); - - // private static final Pattern FREE_DOWNLOAD_URL_PATTERN = Pattern - // .compile("/dl/([0-9]*)/([A-Za-z0-9]*)/([^\"]*)"); - // private static final Pattern DOWNLOAD_ACTION_PATTERN = Pattern - // .compile("name=action value=([A-Za-z0-9]*)"); - // private static final Pattern DOWNLOAD_TM_PATTERN = Pattern - // .compile("name=tm value=([A-Za-z0-9]*)"); - // private static final Pattern DOWNLOAD_TMHASH_PATTERN = Pattern - // .compile("name=tmhash value=([A-Za-z0-9]*)"); - // private static final Pattern DOWNLOAD_WAIT_PATTERN = Pattern - // .compile("name=wait value=([A-Za-z0-9]*)"); - // private static final Pattern DOWNLOAD_WAITHASH_PATTERN = Pattern - // .compile("name=waithash value=([A-Za-z0-9]*)"); - // private static final Pattern DOWNLOAD_UPIDHASH_PATTERN = Pattern - // .compile("name=upidhash value=([A-Za-z0-9]*)"); + .compile("http://hotfile\\.com/dl/([0-9]*)/([A-Za-z0-9]*)/(.*)"); public HotFileService(final HotFileServiceConfiguration configuration) { super(configuration); @@ -146,7 +128,7 @@ public class HotFileService extends @Override public boolean matchURL(URL url) { - return false; + return DOWNLOAD_URL_PATTERN.matcher(url.toString()).matches(); } @Override @@ -170,7 +152,7 @@ public class HotFileService extends private final String filename; private final long filesize; - private Future uploadFuture; + private Future uploadFuture; public HotFileUploader(String filename, long filesize) { super(); @@ -180,20 +162,16 @@ public class HotFileService extends @Override public UploadChannel upload() throws IOException { - final String body = HttpClientUtils.getString(client, - "http://www.hotfile.com/"); - final String url = PatternUtils.find(UPLOAD_URL_PATTERN, body); - - final HttpPost upload = new HttpPost(url); - final MultipartEntity entity = new MultipartEntity(); - upload.setEntity(entity); + final HTMLPage page = getAsPage("http://www.hotfile.com/"); + final String action = page.getFormAction(UPLOAD_URL_PATTERN); final LinkedUploadChannel channel = new LinkedUploadChannel(this, filesize, filename); + final MultipartEntity entity = new MultipartEntity(); - entity.addPart("uploads[]", new UploadListenerContentBody(channel)); + entity.addPart("uploads[]", new LinkedUploadChannelContentBody(channel)); - uploadFuture = HttpClientUtils.executeAsync(client, upload); + uploadFuture = postAsPageAsync(action, entity); while (!channel.isLinked() && !uploadFuture.isDone()) { ThreadUtils.sleep(100); } @@ -203,8 +181,7 @@ public class HotFileService extends @Override public String finish() throws IOException { try { - return PatternUtils.find(DOWNLOAD_URL_PATTERN, - uploadFuture.get()); + return uploadFuture.get().getInputValue(DOWNLOAD_URL_PATTERN); } catch (InterruptedException e) { return null; } catch (ExecutionException e) { @@ -223,10 +200,7 @@ public class HotFileService extends @Override public DownloadChannel download(DownloadListener listener) throws IOException { - final HttpGet request = new HttpGet(url.toString()); - final HttpResponse response = client.execute(request); - final String content = IOUtils.toString(response.getEntity() - .getContent()); + final HTMLPage page = getAsPage(url.toString()); // // try to find timer // final String stringTimer = PatternUtils.find(DOWNLOAD_TIMER, @@ -240,14 +214,12 @@ public class HotFileService extends // + " milliseconds"); // } - final String downloadUrl = PatternUtils.find( - DOWNLOAD_DIRECT_LINK_PATTERN, content, 0); + final String downloadUrl = page + .getLink(DOWNLOAD_DIRECT_LINK_PATTERN); // final String tmHash = PatternUtils.find(DOWNLOAD_TMHASH_PATTERN, // content);F if (downloadUrl != null && downloadUrl.length() > 0) { - final HttpGet downloadRequest = new HttpGet(downloadUrl); - final HttpResponse downloadResponse = client - .execute(downloadRequest); + final HttpResponse downloadResponse = get(downloadUrl); final String filename = FilenameUtils.getName(downloadUrl); long contentLength = getContentLength(downloadResponse); @@ -312,31 +284,25 @@ public class HotFileService extends @Override public void login() throws ClientProtocolException, IOException { - final HttpPost login = new HttpPost( - "http://www.hotfile.com/login.php"); final MultipartEntity entity = new MultipartEntity(); - login.setEntity(entity); entity.addPart("returnto", new StringBody("/index.php")); entity.addPart("user", new StringBody(credential.getUsername())); entity.addPart("pass", new StringBody(credential.getPassword())); - String response = HttpClientUtils.execute(client, login); - if (response.toLowerCase().contains( - credential.getUsername().toLowerCase())) + HTMLPage page = postAsPage("http://www.hotfile.com/login.php", + entity); + final Tag accountTag = page.getTagByID("account"); + if (accountTag == null) throw new AuthenticationInvalidCredentialException(); } @Override public void logout() throws IOException { - final HttpPost logout = new HttpPost( - "http://www.megaupload.com/?c=account"); final MultipartEntity entity = new MultipartEntity(); - logout.setEntity(entity); - entity.addPart("logout", new StringBody("1")); - HttpClientUtils.execute(client, logout); + postAsString("http://www.megaupload.com/?c=account", entity); // TODO check logout status } } diff --git a/src/main/java/com/rogiel/httpchannel/service/impl/MegaUploadService.java b/src/main/java/com/rogiel/httpchannel/service/impl/MegaUploadService.java index 1e5a98e..7c93494 100644 --- a/src/main/java/com/rogiel/httpchannel/service/impl/MegaUploadService.java +++ b/src/main/java/com/rogiel/httpchannel/service/impl/MegaUploadService.java @@ -25,13 +25,10 @@ import java.util.concurrent.Future; import java.util.regex.Pattern; import org.apache.commons.io.FilenameUtils; -import org.apache.commons.io.IOUtils; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; import org.apache.http.NameValuePair; import org.apache.http.client.entity.UrlEncodedFormEntity; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; import org.apache.http.entity.mime.MultipartEntity; import org.apache.http.entity.mime.content.StringBody; import org.apache.http.message.BasicNameValuePair; @@ -50,13 +47,13 @@ import com.rogiel.httpchannel.service.Downloader; import com.rogiel.httpchannel.service.DownloaderCapability; import com.rogiel.httpchannel.service.Service; import com.rogiel.httpchannel.service.UploadChannel; -import com.rogiel.httpchannel.service.UploadListenerContentBody; import com.rogiel.httpchannel.service.UploadService; import com.rogiel.httpchannel.service.Uploader; import com.rogiel.httpchannel.service.UploaderCapability; import com.rogiel.httpchannel.service.channel.InputStreamDownloadChannel; import com.rogiel.httpchannel.service.channel.LinkedUploadChannel; import com.rogiel.httpchannel.service.channel.LinkedUploadChannel.LinkedUploadChannelCloseCallback; +import com.rogiel.httpchannel.service.channel.LinkedUploadChannelContentBody; import com.rogiel.httpchannel.service.config.ServiceConfiguration; import com.rogiel.httpchannel.service.config.ServiceConfigurationProperty; import com.rogiel.httpchannel.service.exception.AuthenticationInvalidCredentialException; @@ -67,6 +64,7 @@ import com.rogiel.httpchannel.service.impl.MegaUploadService.MegaUploadServiceCo import com.rogiel.httpchannel.util.HttpClientUtils; import com.rogiel.httpchannel.util.PatternUtils; import com.rogiel.httpchannel.util.ThreadUtils; +import com.rogiel.httpchannel.util.htmlparser.HTMLPage; /** * This service handles login, upload and download to MegaUpload.com. @@ -81,7 +79,7 @@ public class MegaUploadService extends .compile("http://www([0-9]*)\\.megaupload\\.com/upload_done\\.php\\?UPLOAD_IDENTIFIER=[0-9]*"); private static final Pattern DOWNLOAD_DIRECT_LINK_PATTERN = Pattern - .compile("http://www([0-9]*)\\.megaupload\\.com/files/([A-Za-z0-9]*)/([^\"]*)"); + .compile("http://www([0-9]*)\\.megaupload\\.com/files/([A-Za-z0-9]*)/(.*)"); private static final Pattern DOWNLOAD_TIMER = Pattern .compile("count=([0-9]*);"); // private static final Pattern DOWNLOAD_FILESIZE = Pattern @@ -90,6 +88,9 @@ public class MegaUploadService extends private static final Pattern DOWNLOAD_URL_PATTERN = Pattern .compile("http://www\\.megaupload\\.com/\\?d=([A-Za-z0-9]*)"); + private static final Pattern LOGIN_USERNAME_PATTERN = Pattern + .compile("flashvars\\.username = \"(.*)\";"); + public MegaUploadService(final MegaUploadServiceConfiguration configuration) { super(configuration); } @@ -140,7 +141,7 @@ public class MegaUploadService extends @Override public boolean matchURL(URL url) { - return false; + return DOWNLOAD_URL_PATTERN.matcher(url.toString()).matches(); } @Override @@ -179,22 +180,18 @@ public class MegaUploadService extends @Override public UploadChannel upload() throws IOException { - final String body = HttpClientUtils.getString(client, - "http://www.megaupload.com/multiupload/"); - final String url = PatternUtils.find(UPLOAD_URL_PATTERN, body); - - final HttpPost upload = new HttpPost(url); - final MultipartEntity entity = new MultipartEntity(); - upload.setEntity(entity); + final HTMLPage page = getAsPage("http://www.megaupload.com/multiupload/"); + final String url = page.getFormAction(UPLOAD_URL_PATTERN); final LinkedUploadChannel channel = new LinkedUploadChannel(this, filesize, filename); + final MultipartEntity entity = new MultipartEntity(); - entity.addPart("multifile_0", - new UploadListenerContentBody(channel)); + entity.addPart("multifile_0", new LinkedUploadChannelContentBody( + channel)); entity.addPart("multimessage_0", new StringBody(description)); - uploadFuture = HttpClientUtils.executeAsync(client, upload); + uploadFuture = postAsStringAsync(url, entity); while (!channel.isLinked() && !uploadFuture.isDone()) { ThreadUtils.sleep(100); } @@ -227,42 +224,38 @@ public class MegaUploadService extends @Override public DownloadChannel download(DownloadListener listener) throws IOException { - HttpResponse response = HttpClientUtils.get(client, url.toString()); + HttpResponse response = get(url.toString()); // disable direct downloads, we don't support them! if (response.getEntity().getContentType().getValue() .equals("application/octet-stream")) { - final HttpPost updateAutoDownload = new HttpPost( - "http://www.megaupload.com/?c=account"); + // close connection + response.getEntity().getContent().close(); + final List pairs = new ArrayList(); pairs.add(new BasicNameValuePair("do", "directdownloads")); pairs.add(new BasicNameValuePair("accountupdate", "1")); pairs.add(new BasicNameValuePair("set_ddl", "0")); - updateAutoDownload.setEntity(new UrlEncodedFormEntity(pairs)); - // close connection - response.getEntity().getContent().close(); + // execute update + postAsString("http://www.megaupload.com/?c=account", + new UrlEncodedFormEntity(pairs)); // execute and re-request download - response = HttpClientUtils.get(client, url.toString()); + response = get(url.toString()); } - final String content = IOUtils.toString(response.getEntity() - .getContent()); + final HTMLPage page = HttpClientUtils.toPage(response); // try to find timer - int timer = parseTimer(PatternUtils - .find(DOWNLOAD_TIMER, content, 1)); + int timer = page.findIntegerInScript(DOWNLOAD_TIMER, 1); if (timer > 0 && configuration.respectWaitTime()) { timer(listener, timer * 1000); } - - final String downloadUrl = PatternUtils.find( - DOWNLOAD_DIRECT_LINK_PATTERN, content, 0); + final String downloadUrl = page + .getLink(DOWNLOAD_DIRECT_LINK_PATTERN); if (downloadUrl != null && downloadUrl.length() > 0) { - final HttpGet downloadRequest = new HttpGet(downloadUrl); - final HttpResponse downloadResponse = client - .execute(downloadRequest); + final HttpResponse downloadResponse = get(downloadUrl); if (downloadResponse.getStatusLine().getStatusCode() == HttpStatus.SC_FORBIDDEN || downloadResponse.getStatusLine().getStatusCode() == HttpStatus.SC_SERVICE_UNAVAILABLE) { downloadResponse.getEntity().getContent().close(); @@ -291,31 +284,26 @@ public class MegaUploadService extends @Override public void login() throws IOException { - final HttpPost login = new HttpPost( - "http://www.megaupload.com/?c=login"); final MultipartEntity entity = new MultipartEntity(); - login.setEntity(entity); entity.addPart("login", new StringBody("1")); entity.addPart("username", new StringBody(credential.getUsername())); entity.addPart("password", new StringBody(credential.getPassword())); - final String response = HttpClientUtils.execute(client, login); - if (response.contains("Username and password do " - + "not match. Please try again!")) + final HTMLPage page = postAsPage( + "http://www.megaupload.com/?c=login", entity); + String username = page.findInScript(LOGIN_USERNAME_PATTERN, 1); + + if (username == null) throw new AuthenticationInvalidCredentialException(); } @Override public void logout() throws IOException { - final HttpPost logout = new HttpPost( - "http://www.megaupload.com/?c=account"); final MultipartEntity entity = new MultipartEntity(); - logout.setEntity(entity); - entity.addPart("logout", new StringBody("1")); - HttpClientUtils.execute(client, logout); + postAsString("http://www.megaupload.com/?c=account", entity); // TODO check logout status } } diff --git a/src/main/java/com/rogiel/httpchannel/util/ChannelUtils.java b/src/main/java/com/rogiel/httpchannel/util/ChannelUtils.java new file mode 100644 index 0000000..0993675 --- /dev/null +++ b/src/main/java/com/rogiel/httpchannel/util/ChannelUtils.java @@ -0,0 +1,51 @@ +/* + * This file is part of seedbox . + * + * seedbox is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * seedbox is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with seedbox. If not, see . + */ +package com.rogiel.httpchannel.util; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.WritableByteChannel; + +/** + * @author Rogiel + * + */ +public class ChannelUtils { + public static void copy(ReadableByteChannel in, WritableByteChannel out) + throws IOException { + // First, we need a buffer to hold blocks of copied bytes. + ByteBuffer buffer = ByteBuffer.allocateDirect(32 * 1024); + + // Now loop until no more bytes to read and the buffer is empty + while (in.read(buffer) != -1 || buffer.position() > 0) { + // The read() call leaves the buffer in "fill mode". To prepare + // to write bytes from the bufferwe have to put it in "drain mode" + // by flipping it: setting limit to position and position to zero + buffer.flip(); + + // Now write some or all of the bytes out to the output channel + out.write(buffer); + + // Compact the buffer by discarding bytes that were written, + // and shifting any remaining bytes. This method also + // prepares the buffer for the next call to read() by setting the + // position to the limit and the limit to the buffer capacity. + buffer.compact(); + } + } +} diff --git a/src/main/java/com/rogiel/httpchannel/util/HttpClientUtils.java b/src/main/java/com/rogiel/httpchannel/util/HttpClientUtils.java index d8d88f5..49aa909 100644 --- a/src/main/java/com/rogiel/httpchannel/util/HttpClientUtils.java +++ b/src/main/java/com/rogiel/httpchannel/util/HttpClientUtils.java @@ -29,6 +29,8 @@ import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpUriRequest; +import com.rogiel.httpchannel.util.htmlparser.HTMLPage; + public class HttpClientUtils { private static final ExecutorService threadPool = Executors .newCachedThreadPool(); @@ -77,4 +79,8 @@ public class HttpClientUtils { in.close(); } } + + public static HTMLPage toPage(HttpResponse response) throws IOException { + return HTMLPage.parse(toString(response)); + } } diff --git a/src/main/java/com/rogiel/httpchannel/util/htmlparser/HTMLPage.java b/src/main/java/com/rogiel/httpchannel/util/htmlparser/HTMLPage.java new file mode 100644 index 0000000..18b0ad7 --- /dev/null +++ b/src/main/java/com/rogiel/httpchannel/util/htmlparser/HTMLPage.java @@ -0,0 +1,245 @@ +/* + * This file is part of seedbox . + * + * seedbox is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * seedbox is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with seedbox. If not, see . + */ +package com.rogiel.httpchannel.util.htmlparser; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.htmlparser.Node; +import org.htmlparser.NodeFilter; +import org.htmlparser.Parser; +import org.htmlparser.Tag; +import org.htmlparser.tags.FormTag; +import org.htmlparser.tags.InputTag; +import org.htmlparser.tags.LinkTag; +import org.htmlparser.tags.ScriptTag; +import org.htmlparser.util.NodeIterator; +import org.htmlparser.util.NodeList; +import org.htmlparser.util.ParserException; + +/** + * @author Rogiel + */ +public class HTMLPage { + private final Parser parser; + + private HTMLPage(Parser parser) { + this.parser = parser; + } + + public String getLink(final Pattern pattern) { + NodeList nodes; + try { + nodes = parser.extractAllNodesThatMatch(new NodeFilter() { + private static final long serialVersionUID = 1L; + + @Override + public boolean accept(Node node) { + if (!(node instanceof LinkTag)) + return false; + final LinkTag link = (LinkTag) node; + return pattern.matcher(link.getLink()).matches(); + } + }); + } catch (ParserException e) { + return null; + } + if (nodes.size() >= 1) + return ((LinkTag) nodes.elements().nextNode()).getLink(); + return null; + } + + public String getFormAction(final Pattern pattern) { + NodeList nodes; + try { + nodes = parser.extractAllNodesThatMatch(new NodeFilter() { + private static final long serialVersionUID = 1L; + + @Override + public boolean accept(Node node) { + if (!(node instanceof FormTag)) + return false; + final FormTag form = (FormTag) node; + return pattern.matcher(form.getFormLocation()).matches(); + } + }); + } catch (ParserException e) { + return null; + } + if (nodes.size() >= 1) + return ((FormTag) nodes.elements().nextNode()).getFormLocation(); + return null; + } + + public String getInputValue(final String inputName) { + NodeList nodes; + try { + nodes = parser.extractAllNodesThatMatch(new NodeFilter() { + private static final long serialVersionUID = 1L; + + @Override + public boolean accept(Node node) { + if (!(node instanceof InputTag)) + return false; + final InputTag input = (InputTag) node; + if (!input.getAttribute("name").equals(inputName)) + return false; + return true; + } + }); + } catch (ParserException e) { + return null; + } + if (nodes.size() >= 1) + return ((InputTag) nodes.elements().nextNode()) + .getAttribute("value"); + return null; + } + + public int getIntegerInputValue(final String inputName) { + return Integer.parseInt(getInputValue(inputName)); + } + + public String getInputValue(final Pattern pattern) { + NodeList nodes; + try { + nodes = parser.extractAllNodesThatMatch(new NodeFilter() { + private static final long serialVersionUID = 1L; + + @Override + public boolean accept(Node node) { + if (!(node instanceof InputTag)) + return false; + final InputTag input = (InputTag) node; + if (input.getAttribute("value") == null) + return false; + if (!pattern.matcher(input.getAttribute("value")).matches()) + return false; + return true; + } + }); + } catch (ParserException e) { + return null; + } + if (nodes.size() >= 1) + return ((InputTag) nodes.elements().nextNode()) + .getAttribute("value"); + return null; + } + + public Tag getTagByID(final String id) { + NodeList nodes; + try { + nodes = parser.extractAllNodesThatMatch(new NodeFilter() { + private static final long serialVersionUID = 1L; + + @Override + public boolean accept(Node node) { + if (!(node instanceof Tag)) + return false; + if (((Tag) node).getAttribute("id") == null) + return false; + return ((Tag) node).getAttribute("id").equals(id); + } + }); + } catch (ParserException e) { + return null; + } + if (nodes.size() >= 1) + return ((Tag) nodes.elements().nextNode()); + return null; + } + + public Tag getTagByName(final String name) { + NodeList nodes; + try { + nodes = parser.extractAllNodesThatMatch(new NodeFilter() { + private static final long serialVersionUID = 1L; + + @Override + public boolean accept(Node node) { + if (!(node instanceof Tag)) + return false; + return ((Tag) node).getAttribute("name").equals(name); + } + }); + } catch (ParserException e) { + return null; + } + if (nodes.size() >= 1) + return ((Tag) nodes.elements().nextNode()); + return null; + } + + public boolean contains(final String text) { + try { + for (NodeIterator e = parser.elements(); e.hasMoreNodes();) { + if (e.nextNode().toPlainTextString().contains(text)) + return true; + } + } catch (ParserException e) { + return false; + } + return false; + } + + public String findInScript(final Pattern pattern, int n) { + NodeList nodes; + try { + nodes = parser.extractAllNodesThatMatch(new NodeFilter() { + private static final long serialVersionUID = 1L; + + @Override + public boolean accept(Node node) { + if (!(node instanceof ScriptTag)) + return false; + final ScriptTag script = (ScriptTag) node; + return pattern.matcher(script.getScriptCode()).find(); + } + }); + } catch (ParserException e) { + return null; + } + if (nodes.size() >= 1) { + final ScriptTag script = (ScriptTag) nodes.elements().nextNode(); + final Matcher matcher = pattern.matcher(script.getScriptCode()); + if (matcher.find()) + return matcher.group(n); + } + return null; + } + + public int findIntegerInScript(final Pattern pattern, int n) { + return Integer.parseInt(findInScript(pattern, n)); + } + + public String toString() { + final StringBuilder builder = new StringBuilder(); + try { + for (NodeIterator i = parser.elements(); i.hasMoreNodes();) { + builder.append(i.nextNode().toHtml(true)); + } + } catch (ParserException e) { + return null; + } + return builder.toString(); + } + + public static HTMLPage parse(String html) { + return new HTMLPage(Parser.createParser(html, null)); + } +} diff --git a/src/test/java/com/rogiel/httpchannel/service/impl/HotFileServiceTest.java b/src/test/java/com/rogiel/httpchannel/service/impl/HotFileServiceTest.java index 6de15bb..afd4073 100644 --- a/src/test/java/com/rogiel/httpchannel/service/impl/HotFileServiceTest.java +++ b/src/test/java/com/rogiel/httpchannel/service/impl/HotFileServiceTest.java @@ -20,16 +20,16 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import java.io.ByteArrayOutputStream; -import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; -import java.nio.ByteBuffer; import java.nio.channels.Channels; -import java.nio.channels.FileChannel; -import java.nio.channels.ReadableByteChannel; -import java.nio.channels.WritableByteChannel; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Properties; import junit.framework.Assert; @@ -43,15 +43,18 @@ import com.rogiel.httpchannel.service.Credential; import com.rogiel.httpchannel.service.DownloadChannel; import com.rogiel.httpchannel.service.DownloadService; import com.rogiel.httpchannel.service.Service; +import com.rogiel.httpchannel.service.ServiceHelper; import com.rogiel.httpchannel.service.UploadChannel; import com.rogiel.httpchannel.service.UploadService; import com.rogiel.httpchannel.service.UploaderCapability; import com.rogiel.httpchannel.service.config.ServiceConfigurationHelper; import com.rogiel.httpchannel.service.exception.AuthenticationInvalidCredentialException; import com.rogiel.httpchannel.service.impl.HotFileService.HotFileServiceConfiguration; +import com.rogiel.httpchannel.util.ChannelUtils; public class HotFileServiceTest { private Service service; + private ServiceHelper helper; /** * See src/test/resources/config/hotfile.properties @@ -75,6 +78,7 @@ public class HotFileServiceTest { service = new HotFileService( ServiceConfigurationHelper .defaultConfiguration(HotFileServiceConfiguration.class)); + helper = new ServiceHelper(service); final Properties properties = new Properties(); properties.load(new FileInputStream( @@ -102,21 +106,24 @@ public class HotFileServiceTest { } @Test - public void testNonLoguedInUploader() throws IOException { + public void testNonLoguedInUploader() throws IOException, + URISyntaxException { assertTrue( "This service does not have the capability UploadCapability.FREE_UPLOAD", ((UploadService) service).getUploadCapabilities().has( UploaderCapability.NON_PREMIUM_ACCOUNT_UPLOAD)); - final UploadChannel channel = ((UploadService) service).getUploader( - "simulado_2010_1_res_all.zip", - new File("simulado_2010_1_res_all.zip").length(), null) - .upload(); - final FileChannel fileChannel = new FileInputStream( - "simulado_2010_1_res_all.zip").getChannel(); + final Path path = Paths.get("src/test/resources/upload-test-file.txt"); + final UploadChannel channel = helper.upload(path, + "httpchannel test upload"); + final SeekableByteChannel inChannel = Files.newByteChannel(path); - copy(fileChannel, channel); - channel.close(); + try { + ChannelUtils.copy(inChannel, channel); + } finally { + inChannel.close(); + channel.close(); + } System.out.println(channel.getDownloadLink()); Assert.assertNotNull(channel.getDownloadLink()); @@ -132,16 +139,17 @@ public class HotFileServiceTest { ((AuthenticationService) service).getAuthenticator( new Credential(VALID_USERNAME, VALID_PASSWORD)).login(); - final UploadChannel channel = ((UploadService) service).getUploader( - "simulado_2010_1_res_all.zip", - new File("simulado_2010_1_res_all.zip").length(), null) - .upload(); + final Path path = Paths.get("src/test/resources/upload-test-file.txt"); + final UploadChannel channel = helper.upload(path, + "httpchannel test upload"); + final SeekableByteChannel inChannel = Files.newByteChannel(path); - final FileChannel fileChannel = new FileInputStream( - "simulado_2010_1_res_all.zip").getChannel(); - - copy(fileChannel, channel); - channel.close(); + try { + ChannelUtils.copy(inChannel, channel); + } finally { + inChannel.close(); + channel.close(); + } System.out.println(channel.getDownloadLink()); Assert.assertNotNull(channel.getDownloadLink()); @@ -175,27 +183,4 @@ public class HotFileServiceTest { IOUtils.copy(Channels.newInputStream(channel), bout); System.out.println(bout.size()); } - - public static void copy(ReadableByteChannel in, WritableByteChannel out) - throws IOException { - // First, we need a buffer to hold blocks of copied bytes. - ByteBuffer buffer = ByteBuffer.allocateDirect(32 * 1024); - - // Now loop until no more bytes to read and the buffer is empty - while (in.read(buffer) != -1 || buffer.position() > 0) { - // The read() call leaves the buffer in "fill mode". To prepare - // to write bytes from the bufferwe have to put it in "drain mode" - // by flipping it: setting limit to position and position to zero - buffer.flip(); - - // Now write some or all of the bytes out to the output channel - out.write(buffer); - - // Compact the buffer by discarding bytes that were written, - // and shifting any remaining bytes. This method also - // prepares the buffer for the next call to read() by setting the - // position to the limit and the limit to the buffer capacity. - buffer.compact(); - } - } } diff --git a/src/test/java/com/rogiel/httpchannel/service/impl/MegaUploadServiceTest.java b/src/test/java/com/rogiel/httpchannel/service/impl/MegaUploadServiceTest.java index 9396e7d..ec03e42 100644 --- a/src/test/java/com/rogiel/httpchannel/service/impl/MegaUploadServiceTest.java +++ b/src/test/java/com/rogiel/httpchannel/service/impl/MegaUploadServiceTest.java @@ -25,8 +25,11 @@ import java.io.FileInputStream; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; -import java.nio.ByteBuffer; import java.nio.channels.Channels; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Properties; import junit.framework.Assert; @@ -41,15 +44,18 @@ import com.rogiel.httpchannel.service.DownloadChannel; import com.rogiel.httpchannel.service.DownloadListener; import com.rogiel.httpchannel.service.DownloadService; import com.rogiel.httpchannel.service.Service; +import com.rogiel.httpchannel.service.ServiceHelper; import com.rogiel.httpchannel.service.UploadChannel; import com.rogiel.httpchannel.service.UploadService; import com.rogiel.httpchannel.service.UploaderCapability; import com.rogiel.httpchannel.service.config.ServiceConfigurationHelper; import com.rogiel.httpchannel.service.exception.AuthenticationInvalidCredentialException; import com.rogiel.httpchannel.service.impl.MegaUploadService.MegaUploadServiceConfiguration; +import com.rogiel.httpchannel.util.ChannelUtils; public class MegaUploadServiceTest { private Service service; + private ServiceHelper helper; /** * See src/test/resources/config/megaupload.properties @@ -73,6 +79,7 @@ public class MegaUploadServiceTest { service = new MegaUploadService( ServiceConfigurationHelper .defaultConfiguration(MegaUploadServiceConfiguration.class)); + helper = new ServiceHelper(service); final Properties properties = new Properties(); properties.load(new FileInputStream( @@ -105,24 +112,19 @@ public class MegaUploadServiceTest { "This service does not have the capability UploadCapability.FREE_UPLOAD", ((UploadService) service).getUploadCapabilities().has( UploaderCapability.NON_PREMIUM_ACCOUNT_UPLOAD)); - final UploadChannel channel = ((UploadService) service).getUploader( - "test.bin", 10, "Upload by httpchannel").upload(); - final ByteBuffer buffer = ByteBuffer.allocate(10); - buffer.put((byte) 0x00); - buffer.put((byte) 0x00); - buffer.put((byte) 0x00); - buffer.put((byte) 0x00); - buffer.put((byte) 0x00); - buffer.put((byte) 0x00); - buffer.put((byte) 0x00); - buffer.put((byte) 0x00); - buffer.put((byte) 0x00); - buffer.put((byte) 0x00); - - buffer.flip(); - - channel.write(buffer); - channel.close(); + final Path path = Paths.get("src/test/resources/upload-test-file.txt"); + final UploadChannel channel = helper.upload(path, + "httpchannel test upload"); + final SeekableByteChannel inChannel = Files.newByteChannel(path); + + try { + ChannelUtils.copy(inChannel, channel); + } finally { + inChannel.close(); + channel.close(); + } + + System.out.println(channel.getDownloadLink()); Assert.assertNotNull(channel.getDownloadLink()); } @@ -136,24 +138,19 @@ public class MegaUploadServiceTest { ((AuthenticationService) service).getAuthenticator( new Credential(VALID_USERNAME, VALID_PASSWORD)).login(); - final UploadChannel channel = ((UploadService) service).getUploader( - "test.bin", 10, "Upload by httpchannel").upload(); - final ByteBuffer buffer = ByteBuffer.allocate(10); - buffer.put((byte) 0x00); - buffer.put((byte) 0x00); - buffer.put((byte) 0x00); - buffer.put((byte) 0x00); - buffer.put((byte) 0x00); - buffer.put((byte) 0x00); - buffer.put((byte) 0x00); - buffer.put((byte) 0x00); - buffer.put((byte) 0x00); - buffer.put((byte) 0x00); - - buffer.flip(); - - channel.write(buffer); - channel.close(); + final Path path = Paths.get("src/test/resources/upload-test-file.txt"); + final UploadChannel channel = helper.upload(path, + "httpchannel test upload"); + final SeekableByteChannel inChannel = Files.newByteChannel(path); + + try { + ChannelUtils.copy(inChannel, channel); + } finally { + inChannel.close(); + channel.close(); + } + + System.out.println(channel.getDownloadLink()); Assert.assertNotNull(channel.getDownloadLink()); } diff --git a/src/test/resources/upload-test-file.txt b/src/test/resources/upload-test-file.txt new file mode 100644 index 0000000..3cfccc8 --- /dev/null +++ b/src/test/resources/upload-test-file.txt @@ -0,0 +1,3 @@ +This is a simple upload test file. + +This is for testing purposes only. \ No newline at end of file