1
0
mirror of https://github.com/Rogiel/httpchannel synced 2025-12-06 07:32:50 +00:00

Implements several services and improves API

This commit is contained in:
2012-01-15 18:31:50 -02:00
parent f210afd16a
commit 23f80b50e6
117 changed files with 3741 additions and 1335 deletions

View File

@@ -7,4 +7,7 @@
<relativePath>..</relativePath>
</parent>
<artifactId>httpchannel-service-megaupload</artifactId>
<groupId>com.rogiel.httpchannel.services</groupId>
<name>HttpChannel/Service/MegaUpload</name>
<description>Provides upload and download access to megaupload.com</description>
</project>

View File

@@ -0,0 +1,16 @@
package com.rogiel.httpchannel.service.impl;
import com.rogiel.httpchannel.service.Downloader.DownloaderConfiguration;
public class MegaUploadDownloaderConfiguration implements
DownloaderConfiguration {
private boolean respectWaitTime = true;
public boolean getRespectWaitTime() {
return respectWaitTime;
}
public void setRespectWaitTime(boolean respectWaitTime) {
this.respectWaitTime = respectWaitTime;
}
}

View File

@@ -18,8 +18,6 @@ package com.rogiel.httpchannel.service.impl;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.regex.Pattern;
@@ -27,14 +25,11 @@ import java.util.regex.Pattern;
import org.apache.commons.io.FilenameUtils;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.entity.mime.MultipartEntity;
import org.apache.http.entity.mime.content.StringBody;
import org.apache.http.message.BasicNameValuePair;
import com.rogiel.httpchannel.service.AbstractDownloader;
import com.rogiel.httpchannel.service.AbstractAuthenticator;
import com.rogiel.httpchannel.service.AbstractHttpDownloader;
import com.rogiel.httpchannel.service.AbstractHttpService;
import com.rogiel.httpchannel.service.AbstractUploader;
import com.rogiel.httpchannel.service.AuthenticationService;
import com.rogiel.httpchannel.service.Authenticator;
import com.rogiel.httpchannel.service.AuthenticatorCapability;
@@ -51,37 +46,31 @@ import com.rogiel.httpchannel.service.UploadChannel;
import com.rogiel.httpchannel.service.UploadService;
import com.rogiel.httpchannel.service.Uploader;
import com.rogiel.httpchannel.service.UploaderCapability;
import com.rogiel.httpchannel.service.channel.InputStreamDownloadChannel;
import com.rogiel.httpchannel.service.channel.LinkedUploadChannel;
import com.rogiel.httpchannel.service.channel.LinkedUploadChannel.LinkedUploadChannelCloseCallback;
import com.rogiel.httpchannel.service.channel.LinkedUploadChannelContentBody;
import com.rogiel.httpchannel.service.config.ServiceConfiguration;
import com.rogiel.httpchannel.service.config.ServiceConfigurationHelper;
import com.rogiel.httpchannel.service.config.ServiceConfigurationProperty;
import com.rogiel.httpchannel.service.config.NullAuthenticatorConfiguration;
import com.rogiel.httpchannel.service.exception.AuthenticationInvalidCredentialException;
import com.rogiel.httpchannel.service.exception.DownloadLimitExceededException;
import com.rogiel.httpchannel.service.exception.DownloadLinkNotFoundException;
import com.rogiel.httpchannel.service.exception.UploadLinkNotFoundException;
import com.rogiel.httpchannel.service.impl.MegaUploadService.MegaUploadServiceConfiguration;
import com.rogiel.httpchannel.util.HttpClientUtils;
import com.rogiel.httpchannel.util.PatternUtils;
import com.rogiel.httpchannel.util.ThreadUtils;
import com.rogiel.httpchannel.util.htmlparser.HTMLPage;
/**
* This service handles login, upload and download to MegaUpload.com.
*
* @author Rogiel
* @author <a href="http://www.rogiel.com">Rogiel</a>
* @since 1.0
*/
public class MegaUploadService extends
AbstractHttpService<MegaUploadServiceConfiguration> implements Service,
UploadService, DownloadService, AuthenticationService {
public class MegaUploadService extends AbstractHttpService implements Service,
UploadService<MegaUploadUploaderConfiguration>,
DownloadService<MegaUploadDownloaderConfiguration>,
AuthenticationService<NullAuthenticatorConfiguration> {
/**
* This service ID
*/
public static final ServiceID SERVICE_ID = ServiceID.create("megaupload");
private static final Pattern UPLOAD_URL_PATTERN = Pattern
.compile("http://www([0-9]*)\\.megaupload\\.com/upload_done\\.php\\?UPLOAD_IDENTIFIER=[0-9]*");
@@ -98,11 +87,6 @@ public class MegaUploadService extends
private static final Pattern LOGIN_USERNAME_PATTERN = Pattern
.compile("flashvars\\.username = \"(.*)\";");
public MegaUploadService() {
super(ServiceConfigurationHelper
.defaultConfiguration(MegaUploadServiceConfiguration.class));
}
@Override
public ServiceID getID() {
return SERVICE_ID;
@@ -119,9 +103,21 @@ public class MegaUploadService extends
}
@Override
public Uploader getUploader(String filename, long filesize,
String description) {
return new MegaUploadUploader(filename, filesize, description);
public Uploader<MegaUploadUploaderConfiguration> getUploader(
String filename, long filesize,
MegaUploadUploaderConfiguration configuration) {
return new MegaUploadUploader(filename, filesize, configuration);
}
@Override
public Uploader<MegaUploadUploaderConfiguration> getUploader(
String filename, long filesize) {
return getUploader(filename, filesize, newUploaderConfiguration());
}
@Override
public MegaUploadUploaderConfiguration newUploaderConfiguration() {
return new MegaUploadUploaderConfiguration();
}
@Override
@@ -143,8 +139,19 @@ public class MegaUploadService extends
}
@Override
public Downloader getDownloader(URL url) {
return new MegaUploadDownloader(url);
public Downloader<MegaUploadDownloaderConfiguration> getDownloader(URL url,
MegaUploadDownloaderConfiguration configuration) {
return new MegaUploadDownloader(url, configuration);
}
@Override
public Downloader<MegaUploadDownloaderConfiguration> getDownloader(URL url) {
return getDownloader(url, newDownloaderConfiguration());
}
@Override
public MegaUploadDownloaderConfiguration newDownloaderConfiguration() {
return new MegaUploadDownloaderConfiguration();
}
@Override
@@ -156,13 +163,28 @@ public class MegaUploadService extends
public CapabilityMatrix<DownloaderCapability> getDownloadCapabilities() {
return new CapabilityMatrix<DownloaderCapability>(
DownloaderCapability.UNAUTHENTICATED_DOWNLOAD,
DownloaderCapability.UNAUTHENTICATED_RESUME,
DownloaderCapability.NON_PREMIUM_ACCOUNT_DOWNLOAD,
DownloaderCapability.PREMIUM_ACCOUNT_DOWNLOAD);
DownloaderCapability.NON_PREMIUM_ACCOUNT_RESUME,
DownloaderCapability.PREMIUM_ACCOUNT_DOWNLOAD,
DownloaderCapability.PREMIUM_ACCOUNT_RESUME);
}
@Override
public Authenticator getAuthenticator(Credential credential) {
return new MegaUploadAuthenticator(credential);
public Authenticator<NullAuthenticatorConfiguration> getAuthenticator(
Credential credential, NullAuthenticatorConfiguration configuration) {
return new MegaUploadAuthenticator(credential, configuration);
}
@Override
public Authenticator<NullAuthenticatorConfiguration> getAuthenticator(
Credential credential) {
return getAuthenticator(credential, newAuthenticatorConfiguration());
}
@Override
public NullAuthenticatorConfiguration newAuthenticatorConfiguration() {
return NullAuthenticatorConfiguration.SHARED_INSTANCE;
}
@Override
@@ -170,50 +192,35 @@ public class MegaUploadService extends
return new CapabilityMatrix<AuthenticatorCapability>();
}
protected class MegaUploadUploader implements Uploader,
protected class MegaUploadUploader extends
AbstractUploader<MegaUploadUploaderConfiguration> implements
Uploader<MegaUploadUploaderConfiguration>,
LinkedUploadChannelCloseCallback {
private final String filename;
private final long filesize;
private final String description;
private Future<String> uploadFuture;
public MegaUploadUploader(String filename, long filesize,
String description) {
this.filename = filename;
this.filesize = filesize;
this.description = (description != null ? description
: configuration.getDefaultUploadDescription());
MegaUploadUploaderConfiguration configuration) {
super(filename, filesize, configuration);
}
@Override
public UploadChannel upload() throws IOException {
final HTMLPage page = getAsPage("http://www.megaupload.com/multiupload/");
final String url = page.getFormAction(UPLOAD_URL_PATTERN);
public UploadChannel openChannel() throws IOException {
final HTMLPage page = get("http://www.megaupload.com/multiupload/")
.asPage();
final String url = page.findFormAction(UPLOAD_URL_PATTERN);
final LinkedUploadChannel channel = new LinkedUploadChannel(this,
filesize, filename);
final MultipartEntity entity = new MultipartEntity();
entity.addPart("multifile_0", new LinkedUploadChannelContentBody(
channel));
entity.addPart("multimessage_0", new StringBody(description));
uploadFuture = postAsStringAsync(url, entity);
while (!channel.isLinked() && !uploadFuture.isDone()) {
ThreadUtils.sleep(100);
}
return channel;
final LinkedUploadChannel channel = createLinkedChannel(this);
uploadFuture = multipartPost(url)
.parameter("multimessage_0", configuration.description())
.parameter("multifile_0", channel).asStringAsync();
return waitChannelLink(channel, uploadFuture);
}
@Override
public String finish() throws IOException {
try {
String link = PatternUtils.find(DOWNLOAD_URL_PATTERN,
return PatternUtils.find(DOWNLOAD_URL_PATTERN,
uploadFuture.get());
if (link == null)
throw new UploadLinkNotFoundException();
return link;
} catch (InterruptedException e) {
return null;
} catch (ExecutionException e) {
@@ -222,17 +229,18 @@ public class MegaUploadService extends
}
}
protected class MegaUploadDownloader extends AbstractDownloader {
private final URL url;
public MegaUploadDownloader(URL url) {
this.url = url;
protected class MegaUploadDownloader extends
AbstractHttpDownloader<MegaUploadDownloaderConfiguration> implements
Downloader<MegaUploadDownloaderConfiguration> {
public MegaUploadDownloader(URL url,
MegaUploadDownloaderConfiguration configuration) {
super(url, configuration);
}
@Override
public DownloadChannel download(DownloadListener listener, long position)
throws IOException {
HttpResponse response = get(url.toString());
public DownloadChannel openChannel(DownloadListener listener,
long position) throws IOException {
HttpResponse response = get(url).request();
// disable direct downloads, we don't support them!
if (response.getEntity().getContentType().getValue()
@@ -240,30 +248,28 @@ public class MegaUploadService extends
// close connection
response.getEntity().getContent().close();
final List<NameValuePair> pairs = new ArrayList<NameValuePair>();
pairs.add(new BasicNameValuePair("do", "directdownloads"));
pairs.add(new BasicNameValuePair("accountupdate", "1"));
pairs.add(new BasicNameValuePair("set_ddl", "0"));
// execute update
postAsString("http://www.megaupload.com/?c=account",
new UrlEncodedFormEntity(pairs));
post("http://www.megaupload.com/?c=account")
.parameter("do", "directdownloads")
.parameter("accountupdate", "1")
.parameter("set_ddl", "0").request();
// execute and re-request download
response = get(url.toString());
response = get(url).request();
}
final HTMLPage page = HttpClientUtils.toPage(response);
// try to find timer
int timer = page.findIntegerInScript(DOWNLOAD_TIMER, 1);
if (timer > 0 && configuration.respectWaitTime()) {
int timer = page.findScriptAsInt(DOWNLOAD_TIMER, 1);
if (timer > 0 && configuration.getRespectWaitTime()) {
timer(listener, timer * 1000);
}
final String downloadUrl = page
.getLink(DOWNLOAD_DIRECT_LINK_PATTERN);
.findLink(DOWNLOAD_DIRECT_LINK_PATTERN);
if (downloadUrl != null && downloadUrl.length() > 0) {
final HttpResponse downloadResponse = get(downloadUrl, position);
final HttpResponse downloadResponse = get(downloadUrl)
.position(position).request();
if (downloadResponse.getStatusLine().getStatusCode() == HttpStatus.SC_FORBIDDEN
|| downloadResponse.getStatusLine().getStatusCode() == HttpStatus.SC_SERVICE_UNAVAILABLE) {
downloadResponse.getEntity().getContent().close();
@@ -274,7 +280,7 @@ public class MegaUploadService extends
final String filename = FilenameUtils.getName(downloadUrl);
final long contentLength = getContentLength(downloadResponse);
return new InputStreamDownloadChannel(downloadResponse
return createInputStreamChannel(downloadResponse
.getEntity().getContent(), contentLength, filename);
}
} else {
@@ -283,51 +289,34 @@ public class MegaUploadService extends
}
}
protected class MegaUploadAuthenticator implements Authenticator {
private final Credential credential;
public MegaUploadAuthenticator(Credential credential) {
this.credential = credential;
protected class MegaUploadAuthenticator extends
AbstractAuthenticator<NullAuthenticatorConfiguration> implements
Authenticator<NullAuthenticatorConfiguration> {
public MegaUploadAuthenticator(Credential credential,
NullAuthenticatorConfiguration configuration) {
super(credential, configuration);
}
@Override
public void login() throws IOException {
final MultipartEntity entity = new MultipartEntity();
entity.addPart("login", new StringBody("1"));
entity.addPart("username", new StringBody(credential.getUsername()));
entity.addPart("password", new StringBody(credential.getPassword()));
final HTMLPage page = postAsPage(
"http://www.megaupload.com/?c=login", entity);
String username = page.findInScript(LOGIN_USERNAME_PATTERN, 1);
final HTMLPage page = post("http://www.megaupload.com/?c=login")
.parameter("login", true)
.parameter("username", credential.getUsername())
.parameter("", credential.getPassword()).asPage();
String username = page.findScript(LOGIN_USERNAME_PATTERN, 1);
if (username == null)
throw new AuthenticationInvalidCredentialException();
}
@Override
public void logout() throws IOException {
final MultipartEntity entity = new MultipartEntity();
entity.addPart("logout", new StringBody("1"));
postAsString("http://www.megaupload.com/?c=account", entity);
post("http://www.megaupload.com/?c=account").parameter("logout",
true).request();
// TODO check logout status
}
}
public static interface MegaUploadServiceConfiguration extends
ServiceConfiguration {
@ServiceConfigurationProperty(key = "megaupload.wait", defaultValue = "true")
boolean respectWaitTime();
@ServiceConfigurationProperty(key = "megaupload.port", defaultValue = "80")
int getPreferedDownloadPort();
@ServiceConfigurationProperty(key = "megaupload.description", defaultValue = "Uploaded by seedbox-httpchannel")
String getDefaultUploadDescription();
}
@Override
public String toString() {
return this.getClass().getSimpleName() + " " + getMajorVersion() + "."

View File

@@ -0,0 +1,29 @@
package com.rogiel.httpchannel.service.impl;
import com.rogiel.httpchannel.service.Uploader.DescriptionableUploaderConfiguration;
import com.rogiel.httpchannel.service.Uploader.UploaderConfiguration;
import com.rogiel.httpchannel.service.impl.MegaUploadService.MegaUploadUploader;
/**
* Describes an configuration for an {@link MegaUploadUploader}
*
* @author <a href="http://www.rogiel.com">Rogiel</a>
*/
public class MegaUploadUploaderConfiguration implements UploaderConfiguration,
DescriptionableUploaderConfiguration {
/**
* The upload description
*/
private String description = DescriptionableUploaderConfiguration.DEFAULT_DESCRIPTION;
@Override
public String description() {
return description;
}
@Override
public MegaUploadUploaderConfiguration description(String description) {
this.description = description;
return this;
}
}

View File

@@ -20,7 +20,7 @@ import junit.framework.Assert;
import org.junit.Test;
import com.rogiel.httpchannel.service.Services;
import com.rogiel.httpchannel.service.helper.Services;
/**
* @author <a href="http://www.rogiel.com">Rogiel</a>

View File

@@ -20,10 +20,8 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.channels.Channels;
import java.nio.channels.SeekableByteChannel;
@@ -38,25 +36,18 @@ import org.apache.commons.io.IOUtils;
import org.junit.Before;
import org.junit.Test;
import com.rogiel.httpchannel.service.AuthenticationService;
import com.rogiel.httpchannel.service.Credential;
import com.rogiel.httpchannel.service.DownloadChannel;
import com.rogiel.httpchannel.service.DownloadListener;
import com.rogiel.httpchannel.service.DownloadService;
import com.rogiel.httpchannel.service.Service;
import com.rogiel.httpchannel.service.ServiceHelper;
import com.rogiel.httpchannel.service.ServiceID;
import com.rogiel.httpchannel.service.UploadChannel;
import com.rogiel.httpchannel.service.UploadService;
import com.rogiel.httpchannel.service.UploaderCapability;
import com.rogiel.httpchannel.service.config.ServiceConfigurationHelper;
import com.rogiel.httpchannel.service.exception.AuthenticationInvalidCredentialException;
import com.rogiel.httpchannel.service.impl.MegaUploadService.MegaUploadServiceConfiguration;
import com.rogiel.httpchannel.service.helper.UploadServices;
import com.rogiel.httpchannel.util.ChannelUtils;
public class MegaUploadServiceTest {
private Service service;
private ServiceHelper helper;
private MegaUploadService service;
/**
* See <b>src/test/resources/config/megaupload.properties</b>
@@ -78,7 +69,6 @@ public class MegaUploadServiceTest {
public void setUp() throws Exception {
// MegaUploadServiceConfiguration.class;
service = new MegaUploadService();
helper = new ServiceHelper(service);
final Properties properties = new Properties();
properties.load(new FileInputStream(
@@ -89,19 +79,18 @@ public class MegaUploadServiceTest {
@Test
public void testServiceId() {
System.out.println("Service: " + service.toString());
assertEquals(ServiceID.create("megaupload"), service.getID());
}
@Test
public void testValidAuthenticator() throws IOException {
((AuthenticationService) service).getAuthenticator(
new Credential(VALID_USERNAME, VALID_PASSWORD)).login();
service.getAuthenticator(new Credential(VALID_USERNAME, VALID_PASSWORD))
.login();
}
@Test(expected = AuthenticationInvalidCredentialException.class)
public void testInvalidAuthenticator() throws IOException {
((AuthenticationService) service).getAuthenticator(
service.getAuthenticator(
new Credential(INVALID_USERNAME, INVALID_PASSWORD)).login();
}
@@ -109,11 +98,11 @@ public class MegaUploadServiceTest {
public void testNonLoguedInUploader() throws IOException {
assertTrue(
"This service does not have the capability UploadCapability.FREE_UPLOAD",
((UploadService) service).getUploadCapabilities().has(
service.getUploadCapabilities().has(
UploaderCapability.NON_PREMIUM_ACCOUNT_UPLOAD));
final Path path = Paths.get("src/test/resources/upload-test-file.txt");
final UploadChannel channel = helper.upload(path,
"httpchannel test upload");
final UploadChannel channel = UploadServices.upload(service, path)
.openChannel();
final SeekableByteChannel inChannel = Files.newByteChannel(path);
try {
@@ -131,15 +120,15 @@ public class MegaUploadServiceTest {
public void testLoguedInUploader() throws IOException {
assertTrue(
"This service does not have the capability UploadCapability.PREMIUM_UPLOAD",
((UploadService) service).getUploadCapabilities().has(
service.getUploadCapabilities().has(
UploaderCapability.PREMIUM_ACCOUNT_UPLOAD));
((AuthenticationService) service).getAuthenticator(
new Credential(VALID_USERNAME, VALID_PASSWORD)).login();
service.getAuthenticator(new Credential(VALID_USERNAME, VALID_PASSWORD))
.login();
final Path path = Paths.get("src/test/resources/upload-test-file.txt");
final UploadChannel channel = helper.upload(path,
"httpchannel test upload");
final UploadChannel channel = UploadServices.upload(service, path)
.openChannel();
final SeekableByteChannel inChannel = Files.newByteChannel(path);
try {
@@ -154,10 +143,10 @@ public class MegaUploadServiceTest {
}
@Test
public void testFreeDownloader() throws IOException, MalformedURLException {
final DownloadChannel channel = ((DownloadService) service)
.getDownloader(new URL("http://www.megaupload.com/?d=CVQKJ1KM"))
.download(new DownloadListener() {
public void testFreeDownloader() throws IOException {
final DownloadChannel channel = service.getDownloader(
new URL("http://www.megaupload.com/?d=CVQKJ1KM")).openChannel(
new DownloadListener() {
@Override
public boolean timer(long time) {
System.out.println("Waiting " + time);
@@ -172,14 +161,13 @@ public class MegaUploadServiceTest {
}
@Test
public void testPremiumDownloader() throws IOException,
MalformedURLException {
((AuthenticationService) service).getAuthenticator(
new Credential(VALID_USERNAME, VALID_PASSWORD)).login();
public void testPremiumDownloader() throws IOException {
service.getAuthenticator(new Credential(VALID_USERNAME, VALID_PASSWORD))
.login();
final DownloadChannel channel = ((DownloadService) service)
.getDownloader(new URL("http://www.megaupload.com/?d=CVQKJ1KM"))
.download(new DownloadListener() {
final DownloadChannel channel = service.getDownloader(
new URL("http://www.megaupload.com/?d=CVQKJ1KM")).openChannel(
new DownloadListener() {
@Override
public boolean timer(long time) {
System.out.println("Waiting " + time);
@@ -192,17 +180,18 @@ public class MegaUploadServiceTest {
}
@Test
public void testNoWaitDownloader() throws IOException,
MalformedURLException {
public void testNoWaitDownloader() throws IOException {
service = new MegaUploadService();
service.setServiceConfiguration(ServiceConfigurationHelper.file(
MegaUploadServiceConfiguration.class, new File(
"src/test/resources/megaupload-nowait.properties")));
// service.setServiceConfiguration(ServiceConfigurationHelper.file(
// MegaUploadServiceConfiguration.class, new File(
// "src/test/resources/megaupload-nowait.properties")));
final MegaUploadDownloaderConfiguration config = new MegaUploadDownloaderConfiguration();
config.setRespectWaitTime(false);
@SuppressWarnings("unused")
final DownloadChannel channel = ((DownloadService) service)
.getDownloader(new URL("http://www.megaupload.com/?d=CVQKJ1KM"))
.download(new DownloadListener() {
@SuppressWarnings({ "unused" })
final DownloadChannel channel = service.getDownloader(
new URL("http://www.megaupload.com/?d=CVQKJ1KM"), config)
.openChannel(new DownloadListener() {
@Override
public boolean timer(long time) {
System.out.println("Waiting " + time);