1
0
mirror of https://github.com/Rogiel/httpchannel synced 2025-12-05 23:22:51 +00:00

Add support for Eclipse 3.7 and implement HTMLParser in services

This commit adds support for Eclipse Indigo (3.7), with native
Maven2Eclipse (m2e). It also adds an HTML parser to be used while
uploading and downloading.
This commit is contained in:
2011-09-13 23:40:39 -03:00
parent 5406d14cbf
commit f39d3100c4
16 changed files with 624 additions and 195 deletions

View File

@@ -4,7 +4,7 @@
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources"/>
<classpathentry kind="src" output="target/test-classes" path="src/test/java"/>
<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources"/>
<classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7"/>
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER"/>
<classpathentry kind="output" path="target/classes"/>
</classpath>

View File

@@ -11,12 +11,13 @@
</arguments>
</buildCommand>
<buildCommand>
<name>org.maven.ide.eclipse.maven2Builder</name>
<name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.m2e.core.maven2Nature</nature>
<nature>org.eclipse.jdt.core.javanature</nature>
<nature>org.maven.ide.eclipse.maven2Nature</nature>
</natures>

View File

@@ -1,13 +1,13 @@
#Sun Aug 14 13:47:37 BRT 2011
#Sun Sep 11 18:22:22 BRT 2011
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
org.eclipse.jdt.core.compiler.compliance=1.6
org.eclipse.jdt.core.compiler.compliance=1.7
org.eclipse.jdt.core.compiler.debug.lineNumber=generate
org.eclipse.jdt.core.compiler.debug.localVariable=generate
org.eclipse.jdt.core.compiler.debug.sourceFile=generate
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.source=1.6
org.eclipse.jdt.core.compiler.source=1.7

View File

@@ -0,0 +1,5 @@
#Tue Sep 13 15:42:20 BRT 2011
activeProfiles=
eclipse.preferences.version=1
resolveWorkspaceProjects=true
version=1

28
pom.xml
View File

@@ -6,6 +6,20 @@
<version>1.0.0</version>
<name>Seedbox - HTTP Channel library</name>
<description>Library capable of downloading and uploading files from free servers using channels.</description>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.7</source>
<target>1.7</target>
</configuration>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>junit</groupId>
@@ -21,7 +35,10 @@
<type>jar</type>
<scope>compile</scope>
<exclusions>
<exclusion><groupId>commons-logging</groupId><artifactId>commons-logging</artifactId></exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
@@ -52,5 +69,14 @@
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.htmlparser</groupId>
<artifactId>htmlparser</artifactId>
<version>2.1</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
</dependencies>
<modules>
</modules>
</project>

View File

@@ -16,12 +16,25 @@
*/
package com.rogiel.httpchannel.service;
import java.io.IOException;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.DefaultHttpClient;
import com.rogiel.httpchannel.service.captcha.CaptchaResolver;
import com.rogiel.httpchannel.service.config.ServiceConfiguration;
import com.rogiel.httpchannel.util.AlwaysRedirectStrategy;
import com.rogiel.httpchannel.util.HttpClientUtils;
import com.rogiel.httpchannel.util.htmlparser.HTMLPage;
/**
* Abstract base service for HTTP enabled services.
@@ -31,6 +44,9 @@ import com.rogiel.httpchannel.util.AlwaysRedirectStrategy;
*/
public abstract class AbstractHttpService<T extends ServiceConfiguration>
extends AbstractService<T> implements Service {
private static final ExecutorService threadPool = Executors
.newCachedThreadPool();
/**
* The {@link HttpClient} instance for this service
*/
@@ -49,4 +65,94 @@ public abstract class AbstractHttpService<T extends ServiceConfiguration>
// client.getParams().setIntParameter(ClientPNames.MAX_REDIRECTS, 10);
// client.setRedirectStrategy(new DefaultRedirectStrategy());
}
protected HttpResponse get(String url) throws ClientProtocolException,
IOException {
final HttpGet request = new HttpGet(url);
return client.execute(request);
}
protected String getAsString(String url) throws ClientProtocolException,
IOException {
return HttpClientUtils.toString(get(url));
}
protected HTMLPage getAsPage(String url) throws ClientProtocolException,
IOException {
return HTMLPage.parse(getAsString(url));
}
public Future<HttpResponse> getAsync(final String url) throws IOException {
return threadPool.submit(new Callable<HttpResponse>() {
@Override
public HttpResponse call() throws Exception {
return get(url);
}
});
}
public Future<String> getAsStringAsync(final String url) throws IOException {
return threadPool.submit(new Callable<String>() {
@Override
public String call() throws Exception {
return getAsString(url);
}
});
}
public Future<HTMLPage> getAsPageAsync(final String url) throws IOException {
return threadPool.submit(new Callable<HTMLPage>() {
@Override
public HTMLPage call() throws Exception {
return getAsPage(url);
}
});
}
protected HttpResponse post(String url, HttpEntity entity)
throws ClientProtocolException, IOException {
final HttpPost request = new HttpPost(url);
request.setEntity(entity);
return client.execute(request);
}
protected String postAsString(String url, HttpEntity entity)
throws ClientProtocolException, IOException {
return HttpClientUtils.toString(post(url, entity));
}
protected HTMLPage postAsPage(String url, HttpEntity entity)
throws ClientProtocolException, IOException {
return HTMLPage.parse(postAsString(url, entity));
}
protected Future<HttpResponse> postAsync(final String url,
final HttpEntity entity) throws IOException {
return threadPool.submit(new Callable<HttpResponse>() {
@Override
public HttpResponse call() throws Exception {
return post(url, entity);
}
});
}
protected Future<String> postAsStringAsync(final String url,
final HttpEntity entity) throws IOException {
return threadPool.submit(new Callable<String>() {
@Override
public String call() throws Exception {
return postAsString(url, entity);
}
});
}
protected Future<HTMLPage> postAsPageAsync(final String url,
final HttpEntity entity) throws IOException {
return threadPool.submit(new Callable<HTMLPage>() {
@Override
public HTMLPage call() throws Exception {
return postAsPage(url, entity);
}
});
}
}

View File

@@ -0,0 +1,50 @@
/*
* This file is part of seedbox <github.com/seedbox>.
*
* seedbox is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* seedbox is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with seedbox. If not, see <http://www.gnu.org/licenses/>.
*/
package com.rogiel.httpchannel.service;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
/**
* @author <a href="http://www.rogiel.com">Rogiel</a>
*/
public class ServiceHelper {
private final Service service;
/**
* @param service
* the service
*/
public ServiceHelper(Service service) {
this.service = service;
}
public UploadChannel upload(Path path, String description)
throws IOException {
return ((UploadService) service).getUploader(
path.getFileName().toString(), Files.size(path), description)
.upload();
}
public UploadChannel upload(File file, String description)
throws IOException {
return ((UploadService) service).getUploader(file.getName(),
file.length(), description).upload();
}
}

View File

@@ -14,7 +14,7 @@
* You should have received a copy of the GNU General Public License
* along with seedbox. If not, see <http://www.gnu.org/licenses/>.
*/
package com.rogiel.httpchannel.service;
package com.rogiel.httpchannel.service.channel;
import java.io.IOException;
import java.io.OutputStream;
@@ -25,7 +25,7 @@ import java.nio.channels.WritableByteChannel;
import org.apache.http.entity.mime.content.AbstractContentBody;
import org.apache.http.entity.mime.content.ContentBody;
import com.rogiel.httpchannel.service.channel.LinkedUploadChannel;
import com.rogiel.httpchannel.service.Uploader;
import com.rogiel.httpchannel.util.ThreadUtils;
/**
@@ -34,10 +34,10 @@ import com.rogiel.httpchannel.util.ThreadUtils;
* @author Rogiel
* @since 1.0
*/
public class UploadListenerContentBody extends AbstractContentBody {
public class LinkedUploadChannelContentBody extends AbstractContentBody {
private final LinkedUploadChannel channel;
public UploadListenerContentBody(LinkedUploadChannel channel) {
public LinkedUploadChannelContentBody(LinkedUploadChannel channel) {
super("application/octet-stream");
this.channel = channel;
}

View File

@@ -23,13 +23,11 @@ import java.util.concurrent.Future;
import java.util.regex.Pattern;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.mime.MultipartEntity;
import org.apache.http.entity.mime.content.StringBody;
import org.htmlparser.Tag;
import com.rogiel.httpchannel.service.AbstractDownloader;
import com.rogiel.httpchannel.service.AbstractHttpService;
@@ -45,19 +43,18 @@ import com.rogiel.httpchannel.service.Downloader;
import com.rogiel.httpchannel.service.DownloaderCapability;
import com.rogiel.httpchannel.service.Service;
import com.rogiel.httpchannel.service.UploadChannel;
import com.rogiel.httpchannel.service.UploadListenerContentBody;
import com.rogiel.httpchannel.service.UploadService;
import com.rogiel.httpchannel.service.Uploader;
import com.rogiel.httpchannel.service.UploaderCapability;
import com.rogiel.httpchannel.service.channel.InputStreamDownloadChannel;
import com.rogiel.httpchannel.service.channel.LinkedUploadChannel;
import com.rogiel.httpchannel.service.channel.LinkedUploadChannelContentBody;
import com.rogiel.httpchannel.service.channel.LinkedUploadChannel.LinkedUploadChannelCloseCallback;
import com.rogiel.httpchannel.service.config.ServiceConfiguration;
import com.rogiel.httpchannel.service.exception.AuthenticationInvalidCredentialException;
import com.rogiel.httpchannel.service.impl.HotFileService.HotFileServiceConfiguration;
import com.rogiel.httpchannel.util.HttpClientUtils;
import com.rogiel.httpchannel.util.PatternUtils;
import com.rogiel.httpchannel.util.ThreadUtils;
import com.rogiel.httpchannel.util.htmlparser.HTMLPage;
/**
* This service handles login, upload and download to HotFile.com.
@@ -72,29 +69,14 @@ public class HotFileService extends
.compile("http://u[0-9]*\\.hotfile\\.com/upload\\.cgi\\?[0-9]*");
private static final Pattern DOWNLOAD_DIRECT_LINK_PATTERN = Pattern
.compile("http://hotfile\\.com/get/([0-9]*)/([A-Za-z0-9]*)/([A-Za-z0-9]*)/([^\"]*)");
.compile("http://hotfile\\.com/get/([0-9]*)/([A-Za-z0-9]*)/([A-Za-z0-9]*)/(.*)");
// private static final Pattern DOWNLOAD_TIMER = Pattern
// .compile("timerend=d\\.getTime\\(\\)\\+([0-9]*);");
// private static final Pattern DOWNLOAD_FILESIZE = Pattern
// .compile("[0-9]*(\\.[0-9]*)? (K|M|G)B");
private static final Pattern DOWNLOAD_URL_PATTERN = Pattern
.compile("http://hotfile\\.com/dl/([0-9]*)/([A-Za-z0-9]*)/([^\"]*)");
// private static final Pattern FREE_DOWNLOAD_URL_PATTERN = Pattern
// .compile("/dl/([0-9]*)/([A-Za-z0-9]*)/([^\"]*)");
// private static final Pattern DOWNLOAD_ACTION_PATTERN = Pattern
// .compile("name=action value=([A-Za-z0-9]*)");
// private static final Pattern DOWNLOAD_TM_PATTERN = Pattern
// .compile("name=tm value=([A-Za-z0-9]*)");
// private static final Pattern DOWNLOAD_TMHASH_PATTERN = Pattern
// .compile("name=tmhash value=([A-Za-z0-9]*)");
// private static final Pattern DOWNLOAD_WAIT_PATTERN = Pattern
// .compile("name=wait value=([A-Za-z0-9]*)");
// private static final Pattern DOWNLOAD_WAITHASH_PATTERN = Pattern
// .compile("name=waithash value=([A-Za-z0-9]*)");
// private static final Pattern DOWNLOAD_UPIDHASH_PATTERN = Pattern
// .compile("name=upidhash value=([A-Za-z0-9]*)");
.compile("http://hotfile\\.com/dl/([0-9]*)/([A-Za-z0-9]*)/(.*)");
public HotFileService(final HotFileServiceConfiguration configuration) {
super(configuration);
@@ -146,7 +128,7 @@ public class HotFileService extends
@Override
public boolean matchURL(URL url) {
return false;
return DOWNLOAD_URL_PATTERN.matcher(url.toString()).matches();
}
@Override
@@ -170,7 +152,7 @@ public class HotFileService extends
private final String filename;
private final long filesize;
private Future<String> uploadFuture;
private Future<HTMLPage> uploadFuture;
public HotFileUploader(String filename, long filesize) {
super();
@@ -180,20 +162,16 @@ public class HotFileService extends
@Override
public UploadChannel upload() throws IOException {
final String body = HttpClientUtils.getString(client,
"http://www.hotfile.com/");
final String url = PatternUtils.find(UPLOAD_URL_PATTERN, body);
final HttpPost upload = new HttpPost(url);
final MultipartEntity entity = new MultipartEntity();
upload.setEntity(entity);
final HTMLPage page = getAsPage("http://www.hotfile.com/");
final String action = page.getFormAction(UPLOAD_URL_PATTERN);
final LinkedUploadChannel channel = new LinkedUploadChannel(this,
filesize, filename);
final MultipartEntity entity = new MultipartEntity();
entity.addPart("uploads[]", new UploadListenerContentBody(channel));
entity.addPart("uploads[]", new LinkedUploadChannelContentBody(channel));
uploadFuture = HttpClientUtils.executeAsync(client, upload);
uploadFuture = postAsPageAsync(action, entity);
while (!channel.isLinked() && !uploadFuture.isDone()) {
ThreadUtils.sleep(100);
}
@@ -203,8 +181,7 @@ public class HotFileService extends
@Override
public String finish() throws IOException {
try {
return PatternUtils.find(DOWNLOAD_URL_PATTERN,
uploadFuture.get());
return uploadFuture.get().getInputValue(DOWNLOAD_URL_PATTERN);
} catch (InterruptedException e) {
return null;
} catch (ExecutionException e) {
@@ -223,10 +200,7 @@ public class HotFileService extends
@Override
public DownloadChannel download(DownloadListener listener)
throws IOException {
final HttpGet request = new HttpGet(url.toString());
final HttpResponse response = client.execute(request);
final String content = IOUtils.toString(response.getEntity()
.getContent());
final HTMLPage page = getAsPage(url.toString());
// // try to find timer
// final String stringTimer = PatternUtils.find(DOWNLOAD_TIMER,
@@ -240,14 +214,12 @@ public class HotFileService extends
// + " milliseconds");
// }
final String downloadUrl = PatternUtils.find(
DOWNLOAD_DIRECT_LINK_PATTERN, content, 0);
final String downloadUrl = page
.getLink(DOWNLOAD_DIRECT_LINK_PATTERN);
// final String tmHash = PatternUtils.find(DOWNLOAD_TMHASH_PATTERN,
// content);F
if (downloadUrl != null && downloadUrl.length() > 0) {
final HttpGet downloadRequest = new HttpGet(downloadUrl);
final HttpResponse downloadResponse = client
.execute(downloadRequest);
final HttpResponse downloadResponse = get(downloadUrl);
final String filename = FilenameUtils.getName(downloadUrl);
long contentLength = getContentLength(downloadResponse);
@@ -312,31 +284,25 @@ public class HotFileService extends
@Override
public void login() throws ClientProtocolException, IOException {
final HttpPost login = new HttpPost(
"http://www.hotfile.com/login.php");
final MultipartEntity entity = new MultipartEntity();
login.setEntity(entity);
entity.addPart("returnto", new StringBody("/index.php"));
entity.addPart("user", new StringBody(credential.getUsername()));
entity.addPart("pass", new StringBody(credential.getPassword()));
String response = HttpClientUtils.execute(client, login);
if (response.toLowerCase().contains(
credential.getUsername().toLowerCase()))
HTMLPage page = postAsPage("http://www.hotfile.com/login.php",
entity);
final Tag accountTag = page.getTagByID("account");
if (accountTag == null)
throw new AuthenticationInvalidCredentialException();
}
@Override
public void logout() throws IOException {
final HttpPost logout = new HttpPost(
"http://www.megaupload.com/?c=account");
final MultipartEntity entity = new MultipartEntity();
logout.setEntity(entity);
entity.addPart("logout", new StringBody("1"));
HttpClientUtils.execute(client, logout);
postAsString("http://www.megaupload.com/?c=account", entity);
// TODO check logout status
}
}

View File

@@ -25,13 +25,10 @@ import java.util.concurrent.Future;
import java.util.regex.Pattern;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.mime.MultipartEntity;
import org.apache.http.entity.mime.content.StringBody;
import org.apache.http.message.BasicNameValuePair;
@@ -50,13 +47,13 @@ import com.rogiel.httpchannel.service.Downloader;
import com.rogiel.httpchannel.service.DownloaderCapability;
import com.rogiel.httpchannel.service.Service;
import com.rogiel.httpchannel.service.UploadChannel;
import com.rogiel.httpchannel.service.UploadListenerContentBody;
import com.rogiel.httpchannel.service.UploadService;
import com.rogiel.httpchannel.service.Uploader;
import com.rogiel.httpchannel.service.UploaderCapability;
import com.rogiel.httpchannel.service.channel.InputStreamDownloadChannel;
import com.rogiel.httpchannel.service.channel.LinkedUploadChannel;
import com.rogiel.httpchannel.service.channel.LinkedUploadChannel.LinkedUploadChannelCloseCallback;
import com.rogiel.httpchannel.service.channel.LinkedUploadChannelContentBody;
import com.rogiel.httpchannel.service.config.ServiceConfiguration;
import com.rogiel.httpchannel.service.config.ServiceConfigurationProperty;
import com.rogiel.httpchannel.service.exception.AuthenticationInvalidCredentialException;
@@ -67,6 +64,7 @@ import com.rogiel.httpchannel.service.impl.MegaUploadService.MegaUploadServiceCo
import com.rogiel.httpchannel.util.HttpClientUtils;
import com.rogiel.httpchannel.util.PatternUtils;
import com.rogiel.httpchannel.util.ThreadUtils;
import com.rogiel.httpchannel.util.htmlparser.HTMLPage;
/**
* This service handles login, upload and download to MegaUpload.com.
@@ -81,7 +79,7 @@ public class MegaUploadService extends
.compile("http://www([0-9]*)\\.megaupload\\.com/upload_done\\.php\\?UPLOAD_IDENTIFIER=[0-9]*");
private static final Pattern DOWNLOAD_DIRECT_LINK_PATTERN = Pattern
.compile("http://www([0-9]*)\\.megaupload\\.com/files/([A-Za-z0-9]*)/([^\"]*)");
.compile("http://www([0-9]*)\\.megaupload\\.com/files/([A-Za-z0-9]*)/(.*)");
private static final Pattern DOWNLOAD_TIMER = Pattern
.compile("count=([0-9]*);");
// private static final Pattern DOWNLOAD_FILESIZE = Pattern
@@ -90,6 +88,9 @@ public class MegaUploadService extends
private static final Pattern DOWNLOAD_URL_PATTERN = Pattern
.compile("http://www\\.megaupload\\.com/\\?d=([A-Za-z0-9]*)");
private static final Pattern LOGIN_USERNAME_PATTERN = Pattern
.compile("flashvars\\.username = \"(.*)\";");
public MegaUploadService(final MegaUploadServiceConfiguration configuration) {
super(configuration);
}
@@ -140,7 +141,7 @@ public class MegaUploadService extends
@Override
public boolean matchURL(URL url) {
return false;
return DOWNLOAD_URL_PATTERN.matcher(url.toString()).matches();
}
@Override
@@ -179,22 +180,18 @@ public class MegaUploadService extends
@Override
public UploadChannel upload() throws IOException {
final String body = HttpClientUtils.getString(client,
"http://www.megaupload.com/multiupload/");
final String url = PatternUtils.find(UPLOAD_URL_PATTERN, body);
final HttpPost upload = new HttpPost(url);
final MultipartEntity entity = new MultipartEntity();
upload.setEntity(entity);
final HTMLPage page = getAsPage("http://www.megaupload.com/multiupload/");
final String url = page.getFormAction(UPLOAD_URL_PATTERN);
final LinkedUploadChannel channel = new LinkedUploadChannel(this,
filesize, filename);
final MultipartEntity entity = new MultipartEntity();
entity.addPart("multifile_0",
new UploadListenerContentBody(channel));
entity.addPart("multifile_0", new LinkedUploadChannelContentBody(
channel));
entity.addPart("multimessage_0", new StringBody(description));
uploadFuture = HttpClientUtils.executeAsync(client, upload);
uploadFuture = postAsStringAsync(url, entity);
while (!channel.isLinked() && !uploadFuture.isDone()) {
ThreadUtils.sleep(100);
}
@@ -227,42 +224,38 @@ public class MegaUploadService extends
@Override
public DownloadChannel download(DownloadListener listener)
throws IOException {
HttpResponse response = HttpClientUtils.get(client, url.toString());
HttpResponse response = get(url.toString());
// disable direct downloads, we don't support them!
if (response.getEntity().getContentType().getValue()
.equals("application/octet-stream")) {
final HttpPost updateAutoDownload = new HttpPost(
"http://www.megaupload.com/?c=account");
// close connection
response.getEntity().getContent().close();
final List<NameValuePair> pairs = new ArrayList<NameValuePair>();
pairs.add(new BasicNameValuePair("do", "directdownloads"));
pairs.add(new BasicNameValuePair("accountupdate", "1"));
pairs.add(new BasicNameValuePair("set_ddl", "0"));
updateAutoDownload.setEntity(new UrlEncodedFormEntity(pairs));
// close connection
response.getEntity().getContent().close();
// execute update
postAsString("http://www.megaupload.com/?c=account",
new UrlEncodedFormEntity(pairs));
// execute and re-request download
response = HttpClientUtils.get(client, url.toString());
response = get(url.toString());
}
final String content = IOUtils.toString(response.getEntity()
.getContent());
final HTMLPage page = HttpClientUtils.toPage(response);
// try to find timer
int timer = parseTimer(PatternUtils
.find(DOWNLOAD_TIMER, content, 1));
int timer = page.findIntegerInScript(DOWNLOAD_TIMER, 1);
if (timer > 0 && configuration.respectWaitTime()) {
timer(listener, timer * 1000);
}
final String downloadUrl = PatternUtils.find(
DOWNLOAD_DIRECT_LINK_PATTERN, content, 0);
final String downloadUrl = page
.getLink(DOWNLOAD_DIRECT_LINK_PATTERN);
if (downloadUrl != null && downloadUrl.length() > 0) {
final HttpGet downloadRequest = new HttpGet(downloadUrl);
final HttpResponse downloadResponse = client
.execute(downloadRequest);
final HttpResponse downloadResponse = get(downloadUrl);
if (downloadResponse.getStatusLine().getStatusCode() == HttpStatus.SC_FORBIDDEN
|| downloadResponse.getStatusLine().getStatusCode() == HttpStatus.SC_SERVICE_UNAVAILABLE) {
downloadResponse.getEntity().getContent().close();
@@ -291,31 +284,26 @@ public class MegaUploadService extends
@Override
public void login() throws IOException {
final HttpPost login = new HttpPost(
"http://www.megaupload.com/?c=login");
final MultipartEntity entity = new MultipartEntity();
login.setEntity(entity);
entity.addPart("login", new StringBody("1"));
entity.addPart("username", new StringBody(credential.getUsername()));
entity.addPart("password", new StringBody(credential.getPassword()));
final String response = HttpClientUtils.execute(client, login);
if (response.contains("Username and password do "
+ "not match. Please try again!"))
final HTMLPage page = postAsPage(
"http://www.megaupload.com/?c=login", entity);
String username = page.findInScript(LOGIN_USERNAME_PATTERN, 1);
if (username == null)
throw new AuthenticationInvalidCredentialException();
}
@Override
public void logout() throws IOException {
final HttpPost logout = new HttpPost(
"http://www.megaupload.com/?c=account");
final MultipartEntity entity = new MultipartEntity();
logout.setEntity(entity);
entity.addPart("logout", new StringBody("1"));
HttpClientUtils.execute(client, logout);
postAsString("http://www.megaupload.com/?c=account", entity);
// TODO check logout status
}
}

View File

@@ -0,0 +1,51 @@
/*
* This file is part of seedbox <github.com/seedbox>.
*
* seedbox is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* seedbox is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with seedbox. If not, see <http://www.gnu.org/licenses/>.
*/
package com.rogiel.httpchannel.util;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
/**
* @author <a href="http://www.rogiel.com">Rogiel</a>
*
*/
public class ChannelUtils {
public static void copy(ReadableByteChannel in, WritableByteChannel out)
throws IOException {
// First, we need a buffer to hold blocks of copied bytes.
ByteBuffer buffer = ByteBuffer.allocateDirect(32 * 1024);
// Now loop until no more bytes to read and the buffer is empty
while (in.read(buffer) != -1 || buffer.position() > 0) {
// The read() call leaves the buffer in "fill mode". To prepare
// to write bytes from the bufferwe have to put it in "drain mode"
// by flipping it: setting limit to position and position to zero
buffer.flip();
// Now write some or all of the bytes out to the output channel
out.write(buffer);
// Compact the buffer by discarding bytes that were written,
// and shifting any remaining bytes. This method also
// prepares the buffer for the next call to read() by setting the
// position to the limit and the limit to the buffer capacity.
buffer.compact();
}
}
}

View File

@@ -29,6 +29,8 @@ import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest;
import com.rogiel.httpchannel.util.htmlparser.HTMLPage;
public class HttpClientUtils {
private static final ExecutorService threadPool = Executors
.newCachedThreadPool();
@@ -77,4 +79,8 @@ public class HttpClientUtils {
in.close();
}
}
public static HTMLPage toPage(HttpResponse response) throws IOException {
return HTMLPage.parse(toString(response));
}
}

View File

@@ -0,0 +1,245 @@
/*
* This file is part of seedbox <github.com/seedbox>.
*
* seedbox is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* seedbox is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with seedbox. If not, see <http://www.gnu.org/licenses/>.
*/
package com.rogiel.httpchannel.util.htmlparser;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.Tag;
import org.htmlparser.tags.FormTag;
import org.htmlparser.tags.InputTag;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.tags.ScriptTag;
import org.htmlparser.util.NodeIterator;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;
/**
* @author <a href="http://www.rogiel.com">Rogiel</a>
*/
public class HTMLPage {
private final Parser parser;
private HTMLPage(Parser parser) {
this.parser = parser;
}
public String getLink(final Pattern pattern) {
NodeList nodes;
try {
nodes = parser.extractAllNodesThatMatch(new NodeFilter() {
private static final long serialVersionUID = 1L;
@Override
public boolean accept(Node node) {
if (!(node instanceof LinkTag))
return false;
final LinkTag link = (LinkTag) node;
return pattern.matcher(link.getLink()).matches();
}
});
} catch (ParserException e) {
return null;
}
if (nodes.size() >= 1)
return ((LinkTag) nodes.elements().nextNode()).getLink();
return null;
}
public String getFormAction(final Pattern pattern) {
NodeList nodes;
try {
nodes = parser.extractAllNodesThatMatch(new NodeFilter() {
private static final long serialVersionUID = 1L;
@Override
public boolean accept(Node node) {
if (!(node instanceof FormTag))
return false;
final FormTag form = (FormTag) node;
return pattern.matcher(form.getFormLocation()).matches();
}
});
} catch (ParserException e) {
return null;
}
if (nodes.size() >= 1)
return ((FormTag) nodes.elements().nextNode()).getFormLocation();
return null;
}
public String getInputValue(final String inputName) {
NodeList nodes;
try {
nodes = parser.extractAllNodesThatMatch(new NodeFilter() {
private static final long serialVersionUID = 1L;
@Override
public boolean accept(Node node) {
if (!(node instanceof InputTag))
return false;
final InputTag input = (InputTag) node;
if (!input.getAttribute("name").equals(inputName))
return false;
return true;
}
});
} catch (ParserException e) {
return null;
}
if (nodes.size() >= 1)
return ((InputTag) nodes.elements().nextNode())
.getAttribute("value");
return null;
}
public int getIntegerInputValue(final String inputName) {
return Integer.parseInt(getInputValue(inputName));
}
public String getInputValue(final Pattern pattern) {
NodeList nodes;
try {
nodes = parser.extractAllNodesThatMatch(new NodeFilter() {
private static final long serialVersionUID = 1L;
@Override
public boolean accept(Node node) {
if (!(node instanceof InputTag))
return false;
final InputTag input = (InputTag) node;
if (input.getAttribute("value") == null)
return false;
if (!pattern.matcher(input.getAttribute("value")).matches())
return false;
return true;
}
});
} catch (ParserException e) {
return null;
}
if (nodes.size() >= 1)
return ((InputTag) nodes.elements().nextNode())
.getAttribute("value");
return null;
}
public Tag getTagByID(final String id) {
NodeList nodes;
try {
nodes = parser.extractAllNodesThatMatch(new NodeFilter() {
private static final long serialVersionUID = 1L;
@Override
public boolean accept(Node node) {
if (!(node instanceof Tag))
return false;
if (((Tag) node).getAttribute("id") == null)
return false;
return ((Tag) node).getAttribute("id").equals(id);
}
});
} catch (ParserException e) {
return null;
}
if (nodes.size() >= 1)
return ((Tag) nodes.elements().nextNode());
return null;
}
public Tag getTagByName(final String name) {
NodeList nodes;
try {
nodes = parser.extractAllNodesThatMatch(new NodeFilter() {
private static final long serialVersionUID = 1L;
@Override
public boolean accept(Node node) {
if (!(node instanceof Tag))
return false;
return ((Tag) node).getAttribute("name").equals(name);
}
});
} catch (ParserException e) {
return null;
}
if (nodes.size() >= 1)
return ((Tag) nodes.elements().nextNode());
return null;
}
public boolean contains(final String text) {
try {
for (NodeIterator e = parser.elements(); e.hasMoreNodes();) {
if (e.nextNode().toPlainTextString().contains(text))
return true;
}
} catch (ParserException e) {
return false;
}
return false;
}
public String findInScript(final Pattern pattern, int n) {
NodeList nodes;
try {
nodes = parser.extractAllNodesThatMatch(new NodeFilter() {
private static final long serialVersionUID = 1L;
@Override
public boolean accept(Node node) {
if (!(node instanceof ScriptTag))
return false;
final ScriptTag script = (ScriptTag) node;
return pattern.matcher(script.getScriptCode()).find();
}
});
} catch (ParserException e) {
return null;
}
if (nodes.size() >= 1) {
final ScriptTag script = (ScriptTag) nodes.elements().nextNode();
final Matcher matcher = pattern.matcher(script.getScriptCode());
if (matcher.find())
return matcher.group(n);
}
return null;
}
public int findIntegerInScript(final Pattern pattern, int n) {
return Integer.parseInt(findInScript(pattern, n));
}
public String toString() {
final StringBuilder builder = new StringBuilder();
try {
for (NodeIterator i = parser.elements(); i.hasMoreNodes();) {
builder.append(i.nextNode().toHtml(true));
}
} catch (ParserException e) {
return null;
}
return builder.toString();
}
public static HTMLPage parse(String html) {
return new HTMLPage(Parser.createParser(html, null));
}
}

View File

@@ -20,16 +20,16 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
import java.nio.channels.SeekableByteChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Properties;
import junit.framework.Assert;
@@ -43,15 +43,18 @@ import com.rogiel.httpchannel.service.Credential;
import com.rogiel.httpchannel.service.DownloadChannel;
import com.rogiel.httpchannel.service.DownloadService;
import com.rogiel.httpchannel.service.Service;
import com.rogiel.httpchannel.service.ServiceHelper;
import com.rogiel.httpchannel.service.UploadChannel;
import com.rogiel.httpchannel.service.UploadService;
import com.rogiel.httpchannel.service.UploaderCapability;
import com.rogiel.httpchannel.service.config.ServiceConfigurationHelper;
import com.rogiel.httpchannel.service.exception.AuthenticationInvalidCredentialException;
import com.rogiel.httpchannel.service.impl.HotFileService.HotFileServiceConfiguration;
import com.rogiel.httpchannel.util.ChannelUtils;
public class HotFileServiceTest {
private Service service;
private ServiceHelper helper;
/**
* See <b>src/test/resources/config/hotfile.properties</b>
@@ -75,6 +78,7 @@ public class HotFileServiceTest {
service = new HotFileService(
ServiceConfigurationHelper
.defaultConfiguration(HotFileServiceConfiguration.class));
helper = new ServiceHelper(service);
final Properties properties = new Properties();
properties.load(new FileInputStream(
@@ -102,21 +106,24 @@ public class HotFileServiceTest {
}
@Test
public void testNonLoguedInUploader() throws IOException {
public void testNonLoguedInUploader() throws IOException,
URISyntaxException {
assertTrue(
"This service does not have the capability UploadCapability.FREE_UPLOAD",
((UploadService) service).getUploadCapabilities().has(
UploaderCapability.NON_PREMIUM_ACCOUNT_UPLOAD));
final UploadChannel channel = ((UploadService) service).getUploader(
"simulado_2010_1_res_all.zip",
new File("simulado_2010_1_res_all.zip").length(), null)
.upload();
final FileChannel fileChannel = new FileInputStream(
"simulado_2010_1_res_all.zip").getChannel();
final Path path = Paths.get("src/test/resources/upload-test-file.txt");
final UploadChannel channel = helper.upload(path,
"httpchannel test upload");
final SeekableByteChannel inChannel = Files.newByteChannel(path);
copy(fileChannel, channel);
try {
ChannelUtils.copy(inChannel, channel);
} finally {
inChannel.close();
channel.close();
}
System.out.println(channel.getDownloadLink());
Assert.assertNotNull(channel.getDownloadLink());
@@ -132,16 +139,17 @@ public class HotFileServiceTest {
((AuthenticationService) service).getAuthenticator(
new Credential(VALID_USERNAME, VALID_PASSWORD)).login();
final UploadChannel channel = ((UploadService) service).getUploader(
"simulado_2010_1_res_all.zip",
new File("simulado_2010_1_res_all.zip").length(), null)
.upload();
final Path path = Paths.get("src/test/resources/upload-test-file.txt");
final UploadChannel channel = helper.upload(path,
"httpchannel test upload");
final SeekableByteChannel inChannel = Files.newByteChannel(path);
final FileChannel fileChannel = new FileInputStream(
"simulado_2010_1_res_all.zip").getChannel();
copy(fileChannel, channel);
try {
ChannelUtils.copy(inChannel, channel);
} finally {
inChannel.close();
channel.close();
}
System.out.println(channel.getDownloadLink());
Assert.assertNotNull(channel.getDownloadLink());
@@ -175,27 +183,4 @@ public class HotFileServiceTest {
IOUtils.copy(Channels.newInputStream(channel), bout);
System.out.println(bout.size());
}
public static void copy(ReadableByteChannel in, WritableByteChannel out)
throws IOException {
// First, we need a buffer to hold blocks of copied bytes.
ByteBuffer buffer = ByteBuffer.allocateDirect(32 * 1024);
// Now loop until no more bytes to read and the buffer is empty
while (in.read(buffer) != -1 || buffer.position() > 0) {
// The read() call leaves the buffer in "fill mode". To prepare
// to write bytes from the bufferwe have to put it in "drain mode"
// by flipping it: setting limit to position and position to zero
buffer.flip();
// Now write some or all of the bytes out to the output channel
out.write(buffer);
// Compact the buffer by discarding bytes that were written,
// and shifting any remaining bytes. This method also
// prepares the buffer for the next call to read() by setting the
// position to the limit and the limit to the buffer capacity.
buffer.compact();
}
}
}

View File

@@ -25,8 +25,11 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.SeekableByteChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Properties;
import junit.framework.Assert;
@@ -41,15 +44,18 @@ import com.rogiel.httpchannel.service.DownloadChannel;
import com.rogiel.httpchannel.service.DownloadListener;
import com.rogiel.httpchannel.service.DownloadService;
import com.rogiel.httpchannel.service.Service;
import com.rogiel.httpchannel.service.ServiceHelper;
import com.rogiel.httpchannel.service.UploadChannel;
import com.rogiel.httpchannel.service.UploadService;
import com.rogiel.httpchannel.service.UploaderCapability;
import com.rogiel.httpchannel.service.config.ServiceConfigurationHelper;
import com.rogiel.httpchannel.service.exception.AuthenticationInvalidCredentialException;
import com.rogiel.httpchannel.service.impl.MegaUploadService.MegaUploadServiceConfiguration;
import com.rogiel.httpchannel.util.ChannelUtils;
public class MegaUploadServiceTest {
private Service service;
private ServiceHelper helper;
/**
* See <b>src/test/resources/config/megaupload.properties</b>
@@ -73,6 +79,7 @@ public class MegaUploadServiceTest {
service = new MegaUploadService(
ServiceConfigurationHelper
.defaultConfiguration(MegaUploadServiceConfiguration.class));
helper = new ServiceHelper(service);
final Properties properties = new Properties();
properties.load(new FileInputStream(
@@ -105,24 +112,19 @@ public class MegaUploadServiceTest {
"This service does not have the capability UploadCapability.FREE_UPLOAD",
((UploadService) service).getUploadCapabilities().has(
UploaderCapability.NON_PREMIUM_ACCOUNT_UPLOAD));
final UploadChannel channel = ((UploadService) service).getUploader(
"test.bin", 10, "Upload by httpchannel").upload();
final ByteBuffer buffer = ByteBuffer.allocate(10);
buffer.put((byte) 0x00);
buffer.put((byte) 0x00);
buffer.put((byte) 0x00);
buffer.put((byte) 0x00);
buffer.put((byte) 0x00);
buffer.put((byte) 0x00);
buffer.put((byte) 0x00);
buffer.put((byte) 0x00);
buffer.put((byte) 0x00);
buffer.put((byte) 0x00);
final Path path = Paths.get("src/test/resources/upload-test-file.txt");
final UploadChannel channel = helper.upload(path,
"httpchannel test upload");
final SeekableByteChannel inChannel = Files.newByteChannel(path);
buffer.flip();
channel.write(buffer);
try {
ChannelUtils.copy(inChannel, channel);
} finally {
inChannel.close();
channel.close();
}
System.out.println(channel.getDownloadLink());
Assert.assertNotNull(channel.getDownloadLink());
}
@@ -136,24 +138,19 @@ public class MegaUploadServiceTest {
((AuthenticationService) service).getAuthenticator(
new Credential(VALID_USERNAME, VALID_PASSWORD)).login();
final UploadChannel channel = ((UploadService) service).getUploader(
"test.bin", 10, "Upload by httpchannel").upload();
final ByteBuffer buffer = ByteBuffer.allocate(10);
buffer.put((byte) 0x00);
buffer.put((byte) 0x00);
buffer.put((byte) 0x00);
buffer.put((byte) 0x00);
buffer.put((byte) 0x00);
buffer.put((byte) 0x00);
buffer.put((byte) 0x00);
buffer.put((byte) 0x00);
buffer.put((byte) 0x00);
buffer.put((byte) 0x00);
final Path path = Paths.get("src/test/resources/upload-test-file.txt");
final UploadChannel channel = helper.upload(path,
"httpchannel test upload");
final SeekableByteChannel inChannel = Files.newByteChannel(path);
buffer.flip();
channel.write(buffer);
try {
ChannelUtils.copy(inChannel, channel);
} finally {
inChannel.close();
channel.close();
}
System.out.println(channel.getDownloadLink());
Assert.assertNotNull(channel.getDownloadLink());
}

View File

@@ -0,0 +1,3 @@
This is a simple upload test file.
This is for testing purposes only.