diff --git a/.gitignore b/.gitignore index 84d2250..74e95db 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,8 @@ build/ .gradle/ .idea/ *.iml + +# Eclipse IDE stuff +/.settings +/.project +/.classpath diff --git a/README.md b/README.md index f88f73b..064de68 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,22 @@ # Freenet plugin Keep Alive -#### For building run -```gradle jar``` +### Dependencies +- java8 jdk + - needs to be set as gradle JAVA_HOME +- gradle 4.10.3 (not higher) +- gradle project Freenet REference Daemon (https://github.com/freenet/fred) + - needs to be next to the plugin-KeepAlive folder + - you need to `gradle build` the fred project + +### For building run +``` +gradle jar +``` + +### Build using an IDE +If you would like to use Eclipse than import the project as "Existing Gradle Project" and set the gradle settings as: +- project specific +- specific gradle version to 4.10.3 +- set gradle "Java home" to "*\Eclipse Adoptium\jdk8*" without "\bin" + +Also you need to import "Fred" and do the same diff --git a/bin/.gitignore b/bin/.gitignore new file mode 100644 index 0000000..d97dd0c --- /dev/null +++ b/bin/.gitignore @@ -0,0 +1,2 @@ +/main/ +/default/ diff --git a/build.gradle b/build.gradle index ddea5e0..01dd5e8 100644 --- a/build.gradle +++ b/build.gradle @@ -1,10 +1,14 @@ +/* + Gradle 4.10.3 used +*/ apply plugin: 'java' -sourceCompatibility = 1.7 -targetCompatibility = 1.7 +sourceCompatibility = 1.8 +targetCompatibility = 1.8 repositories { mavenCentral() + maven { url 'https://mvn.freenetproject.org' } } configurations { @@ -12,9 +16,15 @@ configurations { } dependencies { - compile files('../fred/build/libs/freenet.jar') - compile group: 'org.apache.ant', name: 'ant', version: '1.10.5' - extraLibs group: 'com.h2database', name: 'h2', version: '1.4.199' + // Option 1: use source code + compile project(':fred') + + // Option 2: use JAR + //compile files('../fred/build/libs/freenet.jar') + + // only used for TarInputStream + compile group: 'org.apache.ant', name: 'ant', version: '1.10.12' + extraLibs group: 'com.h2database', name: 'h2', version: '2.0.206' configurations.compile.extendsFrom(configurations.extraLibs) } @@ -28,8 +38,7 @@ sourceSets { jar { manifest { - attributes( - 'Plugin-Main-Class': 'keepalive.Plugin') + attributes('Plugin-Main-Class': 'keepalive.Plugin') } from { configurations.extraLibs.collect { it.isDirectory() ? it : zipTree(it) } diff --git a/dev_formater/CleanUp.xml b/dev_formater/CleanUp.xml new file mode 100644 index 0000000..5b32c2d --- /dev/null +++ b/dev_formater/CleanUp.xml @@ -0,0 +1,144 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dev_formater/Formatter.xml b/dev_formater/Formatter.xml new file mode 100644 index 0000000..5461ff8 --- /dev/null +++ b/dev_formater/Formatter.xml @@ -0,0 +1,399 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/settings.gradle b/settings.gradle new file mode 100644 index 0000000..bc1b41f --- /dev/null +++ b/settings.gradle @@ -0,0 +1,2 @@ +include ':fred' +project(':fred').projectDir = new File('../fred') diff --git a/src/keepalive/Plugin.java b/src/keepalive/Plugin.java index 53fb337..785b9f9 100644 --- a/src/keepalive/Plugin.java +++ b/src/keepalive/Plugin.java @@ -18,373 +18,313 @@ */ package keepalive; +import java.io.File; +import java.nio.file.Files; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + import freenet.client.HighLevelSimpleClientImpl; -import freenet.keys.FreenetURI; import freenet.pluginmanager.PluginRespirator; -import keepalive.repository.BlockRepository; -import keepalive.repository.DB; -import keepalive.service.net.Client; +import keepalive.exceptions.DAOException; +import keepalive.model.PropertiesKey; +import keepalive.model.SuccessValues; +import keepalive.repository.IDatabaseDAO; +import keepalive.repository.impl.H2DatabaseDAO; import keepalive.service.reinserter.Reinserter; +import keepalive.urivalues.IUriValue; +import keepalive.urivalues.IUriValuesDAO; +import keepalive.urivalues.impl.PropertiesUriValuesDAO; import keepalive.web.AdminPage; import pluginbase.PluginBase; -import java.io.File; -import java.net.MalformedURLException; -import java.sql.Connection; -import java.sql.Statement; -import java.util.concurrent.*; - +/** + * Mainclass and Startpoint + */ public class Plugin extends PluginBase { - - private static final String version = "0.3.3.11-RW"; - - private Thread reinserterRunner; - private long propSavingTimestamp; - private HighLevelSimpleClientImpl hlsc; - private boolean stackTrace = "true".equals(getProp("stackTrace")); - - public Plugin() { - super("KeepAlive", "KeepAlive", "prop.txt"); - setVersion(version); - addPluginToMenu("KeepAlive", "Reinsert sites and files in the background"); - clearLog(); - } - - @Override - public void runPlugin(PluginRespirator pr) { - super.runPlugin(pr); - try { - hlsc = (HighLevelSimpleClientImpl) pluginContext.node.clientCore.makeClient((short) 5, false, true); - - // migrate from 0.2 to 0.3 - if (getProp("version") == null || !getProp("version").substring(0, 3).equals("0.3")) { - int[] ids = getIds(); - - // remove boost params - for (int aId : ids) { - removeProp("boost_" + aId); - } - - // empty all block list - for (int aId : ids) { - setProp("blocks_" + aId, "?"); - } - - setProp("version", version); - } - - // db migration - // TODO: should be refactored to some standard way - try (Connection connection = DB.getConnection(); - Statement statement = connection.createStatement()) { - String sql = "CREATE TABLE IF NOT EXISTS Block (" + - "uri VARCHAR(256) PRIMARY KEY, " + - "data VARBINARY(32768) not null, " + - "last_access TIMESTAMP DEFAULT CURRENT_TIMESTAMP)"; - statement.executeUpdate(sql); - } catch (Exception e) { - log(e.getMessage(), e); - } - - // initial values - if (getProp("loglevel") == null) setIntProp("loglevel", 1); - if (getProp("ids") == null) setProp("ids", ""); - if (getProp("power") == null) setIntProp("power", 6); - if (getProp("active") == null) setIntProp("active", -1); - if (getProp("splitfile_tolerance") == null) setIntProp("splitfile_tolerance", 66); - if (getProp("splitfile_test_size") == null) setIntProp("splitfile_test_size", 18); - if (getProp("log_links") == null) setIntProp("log_links", 1); - if (getProp("log_utc") == null) setIntProp("log_utc", 1); - if (getIntProp("log_utc") == 1) setTimezoneUTC(); - if (getProp("single_url_timeslot") == null) setIntProp("single_url_timeslot", 4); - if (getProp("stackTrace") == null) setProp("stackTrace", "false"); - saveProp(); - - // build page and menu - addPage(new AdminPage(this, pluginContext.node.clientCore.formPassword)); - addMenuItem("Documentation", "Go to the documentation site", - "/USK@l9wlbjlCA7kfcqzpBsrGtLoAB4-Ro3vZ6q2p9bQ~5es,bGAKUAFF8UryI04sxBKnIQSJWTSa08BDS-8jmVQdE4o,AQACAAE/keepalive/15", true); - - // start reinserter - int activeProp = getIntProp("active"); - if (activeProp != -1) { - startReinserter(activeProp); - } - - } catch (Exception e) { - log("Plugin.runPlugin Exception: " + e.getMessage(), 0); - } - } - - public void startReinserter(final int siteId) { - try { - - // stop previous reinserter - stopReinserter(); - - setIntProp("active", siteId); - saveProp(); - - // start this one - synchronized (this) { - final Plugin plugin = this; - reinserterRunner = new Thread(new Runnable() { - @Override - public void run() { - for (int id = siteId; ; ) { - if (Thread.currentThread().isInterrupted()) { - return; - } - - setIntProp("active", id); - saveProp(); - - CountDownLatch latch = new CountDownLatch(1); - Reinserter reinserter = new Reinserter(plugin, id, latch); - reinserter.start(); - try { - if (!latch.await(getIntProp("single_url_timeslot"), TimeUnit.HOURS)) { - reinserter.interrupt(); - log("Terminated reinserter " + id + " by timeout"); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - reinserter.interrupt(); - return; - } - - // get next siteId - int[] ids = getIds(); - int i = 0; - for (; i < ids.length; i++) { - if (id == ids[i]) { - break; - } - } - if (i < ids.length - 1) { - id = ids[i + 1]; - } else { - id = ids[0]; - } - } - } - }); - reinserterRunner.setName("KeepAlive Reinserter Runner"); - reinserterRunner.start(); - } - - } catch (Exception e) { - log("Plugin.startReinserter Exception: " + e.getMessage(), 0); - } - } - - public synchronized void stopReinserter() { - try { - - if (reinserterRunner != null) { - reinserterRunner.interrupt(); - setIntProp("active", -1); - saveProp(); - } - - } catch (Exception e) { - log("Plugin.stopReinserter Exception: " + e.getMessage(), 0); - } - } - - public int[] getIds() { - try { - - if (getProp("ids") == null || getProp("ids").equals("")) { - return new int[]{}; - } else { - String[] ids = getProp("ids").split(","); - int[] intIds = new int[ids.length]; - for (int i = 0; i < intIds.length; i++) { - intIds[i] = Integer.parseInt(ids[i]); - } - - return intIds; - } - - } catch (Exception e) { - log("Plugin.getIds Exception: " + e.getMessage(), 0); - return null; - } - } - - public int[] getSuccessValues(int siteId) { - try { - - // available blocks - int success = 0; - int failed = 0; - String[] successMap = getProp("success_" + siteId).split(","); - if (successMap.length >= 2) { - for (int i = 0; i < successMap.length; i += 2) { - success += Integer.parseInt(successMap[i]); - failed += Integer.parseInt(successMap[i + 1]); - } - } - - // available segments - int availableSegments = 0; - String successSegments = getProp("success_segments_" + siteId); - int lastTriedSegment = getIntProp("segment_" + siteId); - if (successSegments != null) { - if (lastTriedSegment >= successSegments.length()) { - log("Plugin.getSuccessValues(): List of success_segments too short for siteId " + - siteId + "! " + successSegments.length() + " vs " + lastTriedSegment + 1, 0); - } - - for (int i = 0; i <= lastTriedSegment && i < successSegments.length(); i++) { - if (successSegments.charAt(i) == '1') { - availableSegments++; - } - } - } - - return new int[]{success, failed, availableSegments}; - - } catch (Exception e) { - log("Plugin.getSuccessValues Exception: " + e.getMessage(), 0); - return null; - } - } - - public String getLogFilename(int siteId) { - return "log" + siteId + ".txt"; - } - - public String getBlockListFilename(int siteId) { - return "keys" + siteId + ".txt"; - } - - @Override - public void saveProp() { - if (propSavingTimestamp < System.currentTimeMillis() - 10 * 1000) { - super.saveProp(); - propSavingTimestamp = System.currentTimeMillis(); - } - } - - @Override - public void terminate() { - stopReinserter(); - super.terminate(); - log("plugin terminated", 0); - } - - public HighLevelSimpleClientImpl getFreenetClient() { - return hlsc; - } - - public synchronized boolean isDuplicate(String uri) { - try { - - for (int i : getIds()) { - if (getProp("uri_" + i).equals(uri)) { - return true; - } - } - - } catch (Exception e) { - log("Plugin.isDuplicate Exception: " + e.getMessage(), 2); - } - return false; - } - - public void removeUri(int id) { - // stop reinserter - if (id == getIntProp("active")) { - stopReinserter(); - } - - // remove log and key files - File file = new File(getPluginDirectory() + getLogFilename(id)); - if (file.exists()) { - if (!file.delete()) { - log("Plugin.removeUri(): remove log files was not successful.", 1); - } - } - file = new File(getPluginDirectory() + getBlockListFilename(id)); - if (file.exists()) { - if (!file.delete()) { - log("Plugin.removeUri(): remove key files was not successful.", 1); - } - } - - // remove top block from db - try { - BlockRepository.getInstance(this).delete( - Client.normalizeUri(new FreenetURI(getProp("uri_" + id))).toString()); - } catch (MalformedURLException e) { - log("Can't remove top block from db", e); - } - - // remove items - removeProp("uri_" + id); - removeProp("blocks_" + id); - removeProp("success_" + id); - removeProp("success_segments_" + id); - removeProp("segment_" + id); - removeProp("history_" + id); - String ids = ("," + getProp("ids")).replaceAll("," + id + ",", ","); - setProp("ids", ids.substring(1)); - saveProp(); - } - - @Override - public void setProp(String key, String value) { - try { - super.setProp(key, value); - } catch (Exception e) { - log("Set prop " + key + " " + value, e); - } - } - - @Override - public String getProp(String key) { - try { - return super.getProp(key); - } catch (Exception e) { - log("Get prop " + key, e); - throw new RuntimeException(e); - } - } - - @Override - public void setIntProp(String key, int value) { - try { - super.setIntProp(key, value); - } catch (Exception e) { - log("Set prop " + key + " " + value, e); - } - } - - @Override - public int getIntProp(String key) { - try { - return super.getIntProp(key); - } catch (Exception e) { - log("Get prop " + key, e); - throw new RuntimeException(e); - } - } - - public void log(String info, Throwable e) { - String message = info + ": " + e.getClass().getName() + " " + e.getMessage(); - if (stackTrace) { - message += System.lineSeparator() + stackTraceToString(e); - } - log(message); - } - - private String stackTraceToString(Throwable e) { - StringBuilder sb = new StringBuilder(); - for (StackTraceElement element : e.getStackTrace()) { - sb.append(element.toString()).append(System.lineSeparator()); - } - return sb.toString(); - } + + public static final String VERSION = "0.3.4.3-PlantEater"; + public static final String PLUGIN_NAME = "KeepAlive"; + + private Thread reinserterRunner; + private long propSavingTimestamp; + private HighLevelSimpleClientImpl hlsc; + public IDatabaseDAO databaseDAO; + public IUriValuesDAO uriPropsDAO; + + public Plugin() { + super(PLUGIN_NAME, PLUGIN_NAME, "prop.txt"); + + setVersion(VERSION); + addPluginToMenu(PLUGIN_NAME, "Reinsert sites and files in the background"); + clearLog(); + + this.databaseDAO = new H2DatabaseDAO(this); + this.uriPropsDAO = new PropertiesUriValuesDAO(this, getProperties()); + } + + @Override + public void runPlugin(PluginRespirator pr) { + super.runPlugin(pr); + + try { + hlsc = (HighLevelSimpleClientImpl) pluginContext.node.clientCore.makeClient((short) 5, false, true); + + // initialize all common property keys with its default value + initPropKeys(); + + updateProperties(); + + // db migration + this.databaseDAO.pluginStart(); + + // initial values + if (getIntProp(PropertiesKey.LOG_UTC) == 1) + setTimezoneUTC(); + + // build page and menu + addPage(new AdminPage(this, pluginContext.node.clientCore.formPassword)); + addMenuItem("Documentation", "Go to the documentation site", + "/USK@l9wlbjlCA7kfcqzpBsrGtLoAB4-Ro3vZ6q2p9bQ~5es,bGAKUAFF8UryI04sxBKnIQSJWTSa08BDS-8jmVQdE4o,AQACAAE/keepalive/15", true); + + // start reinserter + final int activeProp = getIntProp(PropertiesKey.ACTIVE); + if (activeProp != -1) { + startReinserter(activeProp); + } + } catch (final Exception e) { + log("Plugin.runPlugin Exception: " + e.getMessage(), 0); + } + } + + private void updateProperties() throws DAOException { + // migrate from 0.2 to 0.3 + if (getProp(PropertiesKey.VERSION) != null && "0.2".equals(getProp(PropertiesKey.VERSION).substring(0, 3))) { + for (final IUriValue uriValue : uriPropsDAO.getAll()) { + // remove boost params + removeProp("boost_" + uriValue.getUriId()); + + // empty all block list + uriValue.setBlockCount(-1); + uriPropsDAO.update(uriValue); + } + + setProp(PropertiesKey.VERSION, VERSION); + saveProp(true); + } + + // rewrite update + if ("0.3.3.11-RW".equals(getProp(PropertiesKey.VERSION))) { + setProp(PropertiesKey.DB_VERSION, "199"); + setProp(PropertiesKey.VERSION, VERSION); + saveProp(true); + return; + } + + // new install + if (getProp(PropertiesKey.VERSION) == null) { + setProp(PropertiesKey.DB_VERSION, "206"); + setProp(PropertiesKey.VERSION, VERSION); + saveProp(true); + } + } + + public void startReinserter(final int siteId) { + try { + // stop previous reinserter + stopReinserter(); + + setIntProp(PropertiesKey.ACTIVE, siteId); + saveProp(); + + // start this one + synchronized (this) { + final Plugin plugin = this; + reinserterRunner = new Thread(() -> { + int startId = siteId; + final Thread thread = Thread.currentThread(); + + while (true) { + try { + for (final IUriValue uriValue : uriPropsDAO.getAll()) { + if (thread.isInterrupted()) + return; + + // find start + if (startId != -1) { + if (uriValue.getUriId() != startId) + continue; + startId = -1; + } + + setIntProp(PropertiesKey.ACTIVE, uriValue.getUriId()); + saveProp(); + + final CountDownLatch latch = new CountDownLatch(1); + final Reinserter reinserter = new Reinserter(plugin, uriValue, latch); + reinserter.start(); + try { + if (!latch.await(getIntProp(PropertiesKey.SINGLE_URL_TIMESLOT), TimeUnit.HOURS)) { + reinserter.interrupt(); + log("Terminated reinserter " + uriValue.getUriId() + " by timeout"); + } + } catch (final InterruptedException e) { + thread.interrupt(); + reinserter.interrupt(); + return; + } + } + } catch (final DAOException e) { + plugin.log("Problem with a DAO", e); + stopReinserter(); + break; + } + } + }); + reinserterRunner.setName(PLUGIN_NAME + " Reinserter Runner"); + reinserterRunner.start(); + } + } catch (final Exception e) { + log("Plugin.startReinserter Exception: " + e.getMessage(), 0); + } + } + + public synchronized void stopReinserter() { + try { + if (reinserterRunner != null) { + reinserterRunner.interrupt(); + setIntProp(PropertiesKey.ACTIVE, -1); + saveProp(); + } + } catch (final Exception e) { + log("Plugin.stopReinserter Exception: " + e.getMessage(), 0); + } + } + + public SuccessValues getSuccessValues(IUriValue uriValue) { + SuccessValues result = new SuccessValues(0, 0, 0); + + try { + // available blocks + int success = 0; + int failed = 0; + final String[] successMap = uriValue.getSuccess().split(","); + if (successMap.length >= 2) { + for (int i = 0; i < successMap.length; i += 2) { + success += Integer.parseInt(successMap[i]); + failed += Integer.parseInt(successMap[i + 1]); + } + } + + // available segments + int availableSegments = 0; + final String successSegments = uriValue.getSuccessSegments(); + final int lastTriedSegment = uriValue.getSegment(); + if (successSegments != null) { + if (lastTriedSegment >= successSegments.length()) { + log("Plugin.getSuccessValues(): List of success_segments too short for siteId " + + uriValue.getUriId() + "! " + successSegments.length() + " vs " + lastTriedSegment + 1, 0); + } + + for (int i = 0; i <= lastTriedSegment && i < successSegments.length(); i++) { + if (successSegments.charAt(i) == '1') { + availableSegments++; + } + } + } + + result = new SuccessValues(success, failed, availableSegments); + } catch (final Exception e) { + log("Plugin.getSuccessValues Exception: " + e.getMessage(), 0); + } + + return result; + } + + public String getLogFilename(IUriValue uriValue) { + return String.format("log%s.txt", uriValue.getUriId()); + } + + public synchronized void saveProp(boolean force) { + if (force || propSavingTimestamp < System.currentTimeMillis() - 10 * 1000) { + super.saveProp(); + propSavingTimestamp = System.currentTimeMillis(); + } + } + + @Override + public void saveProp() { + saveProp(false); + } + + @Override + public void terminate() { + stopReinserter(); + + databaseDAO.pluginTerminate(); + + super.terminate(); + log("plugin terminated", 0); + } + + public HighLevelSimpleClientImpl getFreenetClient() { + return hlsc; + } + + public void removeUriAndFiles(IUriValue uriValue) { + // stop reinserter + if (uriValue.getUriId() == getIntProp(PropertiesKey.ACTIVE)) { + stopReinserter(); + } + + // remove log files + try { + final String fileName = getLogFilename(uriValue); + removeLogFromMap(fileName); + final File file = new File(getPluginDirectory(), fileName); + Files.deleteIfExists(file.toPath()); + } catch (final Exception e) { + log("Plugin.removeUriAndFiles(): remove log files was not successful.", e); + return; + } + + try { + // remove top block from db + databaseDAO.delete(uriValue.getUri()); + + // remove items + uriPropsDAO.delete(uriValue.getUriId()); + } catch (final DAOException e) { + log("Plugin.removeUriAndFiles(): Can't delete uriValues", e); + } + } + + /** + * initialize all common property keys with its default value + */ + private void initPropKeys() { + for (final PropertiesKey key : PropertiesKey.values()) + initPropKey(key); + + saveProp(true); + } + + /** + * initialize a property key with its default value + */ + private void initPropKey(PropertiesKey key) { + if (getProp(key) == null) { + final Object defaultValue = key.getDefault(); + + if (defaultValue == null) { + // value without default ignore + } else if (defaultValue instanceof String) { + setProp(key, (String) defaultValue); + } else if (defaultValue instanceof Integer) { + setIntProp(key, (Integer) defaultValue); + } else { + log(String.format("Unknown default value: '%s' for key: '%s'", defaultValue, key)); + } + } + } + } diff --git a/src/keepalive/exceptions/DAOException.java b/src/keepalive/exceptions/DAOException.java new file mode 100644 index 0000000..4c74a0a --- /dev/null +++ b/src/keepalive/exceptions/DAOException.java @@ -0,0 +1,29 @@ +package keepalive.exceptions; + +public class DAOException extends Exception { + + private static final long serialVersionUID = 87724461226632647L; + + public DAOException() {} + + public DAOException(String message) { + super(message); + } + + public DAOException(String message, Object... args) { + super(String.format(message, args)); + } + + public DAOException(Throwable cause) { + super(cause); + } + + public DAOException(String message, Throwable cause) { + super(message, cause); + } + + public DAOException(String message, Throwable cause, Object... args) { + super(String.format(message, args), cause); + } + +} diff --git a/src/keepalive/exceptions/DatabaseException.java b/src/keepalive/exceptions/DatabaseException.java new file mode 100644 index 0000000..022e3ca --- /dev/null +++ b/src/keepalive/exceptions/DatabaseException.java @@ -0,0 +1,29 @@ +package keepalive.exceptions; + +public class DatabaseException extends Exception { + + private static final long serialVersionUID = 3156450661722048683L; + + public DatabaseException() {} + + public DatabaseException(String message) { + super(message); + } + + public DatabaseException(String message, Object... args) { + super(String.format(message, args)); + } + + public DatabaseException(Throwable cause) { + super(cause); + } + + public DatabaseException(String message, Throwable cause) { + super(message, cause); + } + + public DatabaseException(String message, Throwable cause, Object... args) { + super(String.format(message, args), cause); + } + +} diff --git a/src/keepalive/exceptions/DuplicateKeyException.java b/src/keepalive/exceptions/DuplicateKeyException.java new file mode 100644 index 0000000..c8c2b37 --- /dev/null +++ b/src/keepalive/exceptions/DuplicateKeyException.java @@ -0,0 +1,29 @@ +package keepalive.exceptions; + +public class DuplicateKeyException extends Exception { + + private static final long serialVersionUID = 4112466609861664685L; + + public DuplicateKeyException() {} + + public DuplicateKeyException(String message) { + super(message); + } + + public DuplicateKeyException(String message, Object... args) { + super(String.format(message, args)); + } + + public DuplicateKeyException(Throwable cause) { + super(cause); + } + + public DuplicateKeyException(String message, Throwable cause) { + super(message, cause); + } + + public DuplicateKeyException(String message, Throwable cause, Object... args) { + super(String.format(message, args), cause); + } + +} diff --git a/src/keepalive/exceptions/FetchFailedException.java b/src/keepalive/exceptions/FetchFailedException.java new file mode 100644 index 0000000..52e1101 --- /dev/null +++ b/src/keepalive/exceptions/FetchFailedException.java @@ -0,0 +1,29 @@ +package keepalive.exceptions; + +public class FetchFailedException extends Exception { + + private static final long serialVersionUID = -1746833000014867257L; + + public FetchFailedException() {} + + public FetchFailedException(String message) { + super(message); + } + + public FetchFailedException(String message, Object... args) { + super(String.format(message, args)); + } + + public FetchFailedException(Throwable cause) { + super(cause); + } + + public FetchFailedException(String message, Throwable cause) { + super(message, cause); + } + + public FetchFailedException(String message, Throwable cause, Object... args) { + super(String.format(message, args), cause); + } + +} diff --git a/src/keepalive/exceptions/PropertiesException.java b/src/keepalive/exceptions/PropertiesException.java new file mode 100644 index 0000000..541c327 --- /dev/null +++ b/src/keepalive/exceptions/PropertiesException.java @@ -0,0 +1,29 @@ +package keepalive.exceptions; + +public class PropertiesException extends Exception { + + private static final long serialVersionUID = -2845555980331219730L; + + public PropertiesException() {} + + public PropertiesException(String message) { + super(message); + } + + public PropertiesException(String message, Object... args) { + super(String.format(message, args)); + } + + public PropertiesException(Throwable cause) { + super(cause); + } + + public PropertiesException(String message, Throwable cause) { + super(message, cause); + } + + public PropertiesException(String message, Throwable cause, Object... args) { + super(String.format(message, args), cause); + } + +} diff --git a/src/keepalive/model/Block.java b/src/keepalive/model/Block.java index 549bbda..0582e28 100644 --- a/src/keepalive/model/Block.java +++ b/src/keepalive/model/Block.java @@ -18,94 +18,137 @@ */ package keepalive.model; +import java.util.Objects; + import freenet.keys.FreenetURI; import freenet.support.io.ArrayBucket; -public class Block { - - private int id; - private int segmentId; - private FreenetURI uri; - private ArrayBucket bucket; - private boolean dataBlock; - private boolean fetchDone; // done but not necessarily successful - private boolean fetchSuccessful; - private boolean insertDone; // done but not necessarily successful - private boolean insertSuccessful; - private String resultLog; - - public Block(FreenetURI uri, int segmentId, int id, boolean isDataBlock) { - this.id = id; - this.segmentId = segmentId; - this.uri = uri; - dataBlock = isDataBlock; - } - - public int getId() { - return id; - } - - public int getSegmentId() { - return segmentId; - } - - public FreenetURI getUri() { - return uri; - } - - public ArrayBucket getBucket() { - return bucket; - } - - public void setBucket(ArrayBucket bucket) { - this.bucket = bucket; - } - - public boolean isDataBlock() { - return dataBlock; - } - - public boolean isFetchInProcess() { - return !fetchDone; - } - - public void setFetchDone(boolean done) { - fetchDone = done; - } - - boolean isInsertDone() { - return insertDone; - } - - public void setInsertDone(boolean done) { - insertDone = done; - } - - public boolean isInsertSuccessful() { - return insertSuccessful; - } - - public void setInsertSuccessful(boolean successful) { - insertSuccessful = successful; - } - - public boolean isFetchSuccessful() { - return fetchSuccessful; - } - - public void setFetchSuccessful(boolean successful) { - fetchSuccessful = successful; - } - - public String getResultLog() { - return resultLog; - } - - public void setResultLog(String result) { - resultLog = result; - } - - public void appendResultLog(String result) { - resultLog += result; - } +public class Block implements IBlock { + + private final int id; + private final int segmentId; + private final FreenetURI uri; + private ArrayBucket bucket; + private final boolean dataBlock; + private boolean fetchDone; // done but not necessarily successful + private boolean fetchSuccessful; + private boolean insertDone; // done but not necessarily successful + private boolean insertSuccessful; + private String resultLog; + + public Block(FreenetURI uri, int segmentId, int id, boolean isDataBlock) { + this.uri = uri; + this.segmentId = segmentId; + this.id = id; + this.dataBlock = isDataBlock; + } + + @Override + public int getId() { + return id; + } + + @Override + public int getSegmentId() { + return segmentId; + } + + @Override + public FreenetURI getUri() { + return uri; + } + + @Override + public ArrayBucket getBucket() { + return bucket; + } + + @Override + public void setBucket(ArrayBucket bucket) { + this.bucket = bucket; + } + + @Override + public boolean isDataBlock() { + return dataBlock; + } + + @Override + public boolean isFetchInProcess() { + return !fetchDone; + } + + @Override + public void setFetchDone(boolean done) { + fetchDone = done; + } + + @Override + public boolean isInsertDone() { + return insertDone; + } + + @Override + public void setInsertDone(boolean done) { + insertDone = done; + } + + @Override + public boolean isInsertSuccessful() { + return insertSuccessful; + } + + @Override + public void setInsertSuccessful(boolean successful) { + insertSuccessful = successful; + } + + @Override + public boolean isFetchSuccessful() { + return fetchSuccessful; + } + + @Override + public void setFetchSuccessful(boolean successful) { + fetchSuccessful = successful; + } + + @Override + public String getResultLog() { + return resultLog; + } + + @Override + public void setResultLog(String result) { + resultLog = result; + } + + @Override + public void appendResultLog(String result) { + resultLog += result; + } + + @Override + public String toString() { + return String.format("Block [id=%s, segmentId=%s, uri=%s, bucket=%s, dataBlock=%s, fetchDone=%s, fetchSuccessful=%s, insertDone=%s, insertSuccessful=%s, resultLog=%s]", id, segmentId, uri, + bucket, dataBlock, fetchDone, fetchSuccessful, insertDone, insertSuccessful, resultLog); + } + + @Override + public int hashCode() { + return Objects.hash(bucket, dataBlock, fetchDone, fetchSuccessful, id, insertDone, insertSuccessful, resultLog, segmentId, uri); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if ((obj == null) || (getClass() != obj.getClass())) + return false; + final Block other = (Block) obj; + return Objects.equals(bucket, other.bucket) && dataBlock == other.dataBlock && fetchDone == other.fetchDone && fetchSuccessful == other.fetchSuccessful && id == other.id + && insertDone == other.insertDone && insertSuccessful == other.insertSuccessful && Objects.equals(resultLog, other.resultLog) && segmentId == other.segmentId + && Objects.equals(uri, other.uri); + } + } diff --git a/src/keepalive/model/IBlock.java b/src/keepalive/model/IBlock.java new file mode 100644 index 0000000..b3d9698 --- /dev/null +++ b/src/keepalive/model/IBlock.java @@ -0,0 +1,42 @@ +package keepalive.model; + +import freenet.keys.FreenetURI; +import freenet.support.io.ArrayBucket; + +public interface IBlock { + + int getId(); + + int getSegmentId(); + + FreenetURI getUri(); + + ArrayBucket getBucket(); + + void setBucket(ArrayBucket bucket); + + boolean isDataBlock(); + + boolean isFetchInProcess(); + + void setFetchDone(boolean done); + + boolean isInsertDone(); + + void setInsertDone(boolean done); + + boolean isInsertSuccessful(); + + void setInsertSuccessful(boolean successful); + + boolean isFetchSuccessful(); + + void setFetchSuccessful(boolean successful); + + String getResultLog(); + + void setResultLog(String result); + + void appendResultLog(String result); + +} diff --git a/src/keepalive/model/PropertiesKey.java b/src/keepalive/model/PropertiesKey.java new file mode 100644 index 0000000..6f4dbbb --- /dev/null +++ b/src/keepalive/model/PropertiesKey.java @@ -0,0 +1,70 @@ +/* + * Keep Alive Plugin + * Copyright (C) 2022 PlantEater + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package keepalive.model; + +/** + * enum for all property keys with default values
+ * they get automatically initialised + */ +public enum PropertiesKey { + + // common properties + DB_VERSION("db_version"), //, "199" + VERSION("version"), //, Plugin.VERSION + LOGLEVEL("loglevel", 1), + IDS("ids", ""), + POWER("power", 6), + ACTIVE("active", -1), + SPLITFILE_TOLERANCE("splitfile_tolerance", 66), + SPLITFILE_TEST_SIZE("splitfile_test_size", 18), + LOG_LINKS("log_links", 1), + LOG_UTC("log_utc", 1), + SINGLE_URL_TIMESLOT("single_url_timeslot", 4), + STACKTRACE("stackTrace", "true"), + + // uri specific + URI("uri"), + BLOCKS("blocks"), + SUCCESS_SEGMENTS("success_segments"), + SUCCESS("success"), + HISTORY("history"), + SEGMENT("segment"); + + private final String key; + private final Object defaultValue; + + PropertiesKey(String key, Object defaultValue) { + this.key = key; + this.defaultValue = defaultValue; + } + + PropertiesKey(String key) { + this(key, null); + } + + @Override + public String toString() { + return this.key; + } + + public Object getDefault() { + return this.defaultValue; + } + +} diff --git a/src/keepalive/model/Segment.java b/src/keepalive/model/Segment.java index 58cbbb7..d5ad1c4 100644 --- a/src/keepalive/model/Segment.java +++ b/src/keepalive/model/Segment.java @@ -18,116 +18,149 @@ */ package keepalive.model; +import java.util.Arrays; +import java.util.Objects; + +import keepalive.exceptions.DAOException; import keepalive.service.reinserter.Reinserter; public class Segment { - - private int id; - private final int size; - private Block[] blocks; - private int dataBlocksCount; - private int success = 0; - private int failed = 0; - private boolean persistenceCheckOk = false; - private boolean healingNotPossible = false; - - private final Reinserter reinserter; - - public Segment(Reinserter reinserter, int id, int size) { - this.reinserter = reinserter; - this.id = id; - this.size = size; - blocks = new Block[size]; - } - - public Block getBlock(int id) { - return blocks[id]; - } - - public void addBlock(Block block) { - blocks[block.getId()] = block; - if (block.isDataBlock()) { - dataBlocksCount++; - } - } - - public Block getDataBlock(int id) { - return blocks[id]; - } - - public Block getCheckBlock(int id) { - return blocks[dataSize() + id]; - } - - public int size() { - return size; // blocks.length can produce null-exception (see isFinished()) - } - - public int dataSize() { - return dataBlocksCount; - } - - public int checkSize() { - return size - dataBlocksCount; - } - - public void initInsert() { - success = 0; - failed = 0; - } - - public void regFetchSuccess(double persistenceRate) { - persistenceCheckOk = true; - success = (int) Math.round(persistenceRate * size); - failed = size - success; - reinserter.updateBlockStatistic(id, success, failed); - } - - public void regFetchSuccess(boolean isSuccess) { - if (isSuccess) - success++; - else - failed++; - - reinserter.updateBlockStatistic(id, success, failed); - } - - public int getId() { - return id; - } - - public boolean isFinished() { - if (blocks == null) - return true; - - boolean finished = true; - if (!persistenceCheckOk && !healingNotPossible) { - if (size == 1) { - finished = getBlock(0).isInsertDone(); - } else { - for (int i = 0; i < size; i++) { - if (!getBlock(i).isFetchSuccessful() && !getBlock(i).isInsertDone()) { - finished = false; - break; - } - } - } - } - - // free blocks (especially buckets) - if (finished) { - for (int i = 0; i < size; i++) { - if (blocks[i].getBucket() != null) { - blocks[i].getBucket().free(); - } - } - blocks = null; - } - - return finished; - } - - public void setHealingNotPossible(boolean notPossible) { - healingNotPossible = notPossible; - } + + private final int id; + private final int size; + private IBlock[] blocks; + private int dataBlocksCount; + private int success = 0; + private int failed = 0; + private boolean persistenceCheckOk = false; + private boolean healingNotPossible = false; + + private final Reinserter reinserter; + + public Segment(Reinserter reinserter, int id, int size) { + this.reinserter = reinserter; + this.id = id; + this.size = size; + this.blocks = new IBlock[size]; + } + + public IBlock getBlock(int id) { + return blocks[id]; + } + + public boolean addBlock(IBlock block) { + if (block.getId() > size) + return false; + + blocks[block.getId()] = block; + if (block.isDataBlock()) + dataBlocksCount++; + + return true; + } + + public IBlock getDataBlock(int id) { + return blocks[id]; + } + + public IBlock getCheckBlock(int id) { + return blocks[dataSize() + id]; + } + + public int size() { + return size; // blocks.length can produce null-exception (see isFinished()) + } + + public int dataSize() { + return dataBlocksCount; + } + + public int checkSize() { + return size - dataBlocksCount; + } + + public void initInsert() { + success = 0; + failed = 0; + } + + public void regFetchSuccess(double persistenceRate) throws DAOException { + persistenceCheckOk = true; + success = (int) Math.round(persistenceRate * size); + failed = size - success; + reinserter.updateBlockStatistic(id, success, failed); + } + + public void regFetchSuccess(boolean isSuccess) throws DAOException { + if (isSuccess) + success++; + else + failed++; + + reinserter.updateBlockStatistic(id, success, failed); + } + + public int getId() { + return id; + } + + public boolean isFinished() { + if (blocks == null) + return true; + + boolean finished = true; + if (!persistenceCheckOk && !healingNotPossible) { + if (size == 1) { + finished = getBlock(0).isInsertDone(); + } else { + for (final IBlock block : blocks) { + if (block != null && !block.isFetchSuccessful() && !block.isInsertDone()) { + finished = false; + break; + } + } + } + } + + // free blocks (especially buckets) + if (finished) { + for (final IBlock block : blocks) { + if (block != null && block.getBucket() != null) + block.getBucket().free(); + } + blocks = null; + } + + return finished; + } + + public void setHealingNotPossible(boolean notPossible) { + healingNotPossible = notPossible; + } + + @Override + public String toString() { + return String.format("Segment [id=%s, size=%s, blocks=%s, dataBlocksCount=%s, success=%s, failed=%s, persistenceCheckOk=%s, healingNotPossible=%s, reinserter=%s]", id, size, + Arrays.toString(blocks), dataBlocksCount, success, failed, persistenceCheckOk, healingNotPossible, reinserter); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + Arrays.hashCode(blocks); + return prime * result + Objects.hash(dataBlocksCount, failed, healingNotPossible, id, persistenceCheckOk, reinserter, size, success); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if ((obj == null) || (getClass() != obj.getClass())) + return false; + final Segment other = (Segment) obj; + return Arrays.equals(blocks, other.blocks) && dataBlocksCount == other.dataBlocksCount && failed == other.failed && healingNotPossible == other.healingNotPossible && id == other.id + && persistenceCheckOk == other.persistenceCheckOk && Objects.equals(reinserter, other.reinserter) && size == other.size && success == other.success; + } + } diff --git a/src/keepalive/model/SuccessValues.java b/src/keepalive/model/SuccessValues.java new file mode 100644 index 0000000..6fcdf60 --- /dev/null +++ b/src/keepalive/model/SuccessValues.java @@ -0,0 +1,49 @@ +package keepalive.model; + +import java.util.Objects; + +public class SuccessValues { + + private final int success; + private final int failed; + private final int availableSegments; + + public SuccessValues(int success, int failed, int availableSegments) { + this.success = success; + this.failed = failed; + this.availableSegments = availableSegments; + } + + public int getSuccess() { + return success; + } + + public int getFailed() { + return failed; + } + + public int getAvailableSegments() { + return availableSegments; + } + + @Override + public String toString() { + return String.format("SuccessValues [success=%s, failed=%s, availableSegments=%s]", success, failed, availableSegments); + } + + @Override + public int hashCode() { + return Objects.hash(availableSegments, failed, success); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if ((obj == null) || (getClass() != obj.getClass())) + return false; + final SuccessValues other = (SuccessValues) obj; + return availableSegments == other.availableSegments && failed == other.failed && success == other.success; + } + +} diff --git a/src/keepalive/model/UiKey.java b/src/keepalive/model/UiKey.java new file mode 100644 index 0000000..e664dce --- /dev/null +++ b/src/keepalive/model/UiKey.java @@ -0,0 +1,36 @@ +package keepalive.model; + +public enum UiKey { + + // Fields + FORMPASSWORD("formPassword"), + URIS("uris"), + REMOVE_REGEX("remove_regex"), + + // Buttons + MODIFY_LOGLEVEL("modify_loglevel"), + CLEAR_LOGS("clear_logs"), + CLEAR_HISTORY("clear_history"), + REMOVE("remove"), + REMOVE_WITH_REGEX("remove_with_regex"), + REMOVE_ALL("remove_all"), + MASTER_LOG("master_log"), + LOG("log"), + START("start"), + STOP("stop"), + + // Unknown + SHOW_LOG("show_log"); + + private final String key; + + UiKey(String key) { + this.key = key; + } + + @Override + public String toString() { + return this.key; + } + +} diff --git a/src/keepalive/repository/BlockRepository.java b/src/keepalive/repository/BlockRepository.java deleted file mode 100644 index 1ed85a5..0000000 --- a/src/keepalive/repository/BlockRepository.java +++ /dev/null @@ -1,122 +0,0 @@ -package keepalive.repository; - -import keepalive.Plugin; - -import java.sql.*; - -public class BlockRepository { - - private final Plugin plugin; - - private static BlockRepository instance; - - private static final String SQL_SAVE = "INSERT INTO Block (uri, data) VALUES (?, ?)"; - private static final String SQL_FIND = "SELECT data FROM Block WHERE uri = ?"; - private static final String SQL_UPDATE = "UPDATE Block SET data = ? WHERE uri = ?"; - private static final String SQL_DELETE = "DELETE FROM Block WHERE uri = ?;"; - private static final String SQL_LAST_ACCESS_DIFF = "SELECT TIMESTAMPDIFF(MILLISECOND, last_access, CURRENT_TIMESTAMP) FROM Block WHERE uri = ?"; - private static final String SQL_LAST_ACCESS_UPDATE = "UPDATE Block SET last_access = CURRENT_TIMESTAMP WHERE uri = ?"; - - private BlockRepository(Plugin plugin) { - this.plugin = plugin; - } - - public static synchronized BlockRepository getInstance(Plugin plugin) { - if (instance == null) { - instance = new BlockRepository(plugin); - } - return instance; - } - - public void saveOrUpdate(String uri, byte[] data) { - try (Connection connection = DB.getConnection(); - PreparedStatement findPreparedStatement = connection.prepareStatement(SQL_FIND)) { - findPreparedStatement.setString(1, uri); - ResultSet resultSet = findPreparedStatement.executeQuery(); - - if (resultSet.next()) { - try (PreparedStatement updatePreparedStatement = connection.prepareStatement(SQL_UPDATE)) { - updatePreparedStatement.setBytes(1, data); - updatePreparedStatement.setString(2, uri); - updatePreparedStatement.executeUpdate(); - } - } else { - try (PreparedStatement savePreparedStatement = connection.prepareStatement(SQL_SAVE)) { - savePreparedStatement.setString(1, uri); - savePreparedStatement.setBytes(2, data); - savePreparedStatement.executeUpdate(); - } - } - } catch (SQLException e) { - plugin.log(e.getMessage(), e); - } - } - - public byte[] findOne(String uri) { - try (Connection connection = DB.getConnection(); - PreparedStatement preparedStatement = connection.prepareStatement(SQL_FIND)) { - preparedStatement.setString(1, uri); - ResultSet resultSet = preparedStatement.executeQuery(); - if (resultSet.next()) { - byte[] data = resultSet.getBytes("data"); - - if (resultSet.next()) { - plugin.log("Not unique uri: " + uri); - return null; - } - - return data; - } else { - return null; - } - } catch (SQLException e) { - plugin.log(e.getMessage() + " " + uri, e); - } - - return null; - } - - public void delete(String uri) { - try (Connection connection = DB.getConnection(); - PreparedStatement preparedStatement = connection.prepareStatement(SQL_DELETE)) { - preparedStatement.setString(1, uri); - preparedStatement.executeUpdate(); - } catch (SQLException e) { - plugin.log(e.getMessage() + " " + uri, e); - } - } - - public long lastAccessDiff(String uri) { - try (Connection connection = DB.getConnection(); - PreparedStatement preparedStatement = connection.prepareStatement(SQL_LAST_ACCESS_DIFF)) { - preparedStatement.setString(1, uri); - ResultSet resultSet = preparedStatement.executeQuery(); - if (resultSet.next()) { - long diff = resultSet.getLong(1); - - if (resultSet.next()) { - plugin.log("Not unique uri: " + uri); - return 0; - } - - return diff; - } else { - return 0; - } - } catch (SQLException e) { - plugin.log(e.getMessage() + " " + uri, e); - } - - return 0; - } - - public void lastAccessUpdate(String uri) { - try (Connection connection = DB.getConnection(); - PreparedStatement preparedStatement = connection.prepareStatement(SQL_LAST_ACCESS_UPDATE)) { - preparedStatement.setString(1, uri); - preparedStatement.executeUpdate(); - } catch (SQLException e) { - plugin.log(e.getMessage() + " " + uri, e); - } - } -} diff --git a/src/keepalive/repository/DB.java b/src/keepalive/repository/DB.java deleted file mode 100644 index 61560c6..0000000 --- a/src/keepalive/repository/DB.java +++ /dev/null @@ -1,31 +0,0 @@ -package keepalive.repository; - -import java.io.File; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; - -public class DB { - - private static final String JDBC_DRIVER = "org.h2.Driver"; - private static final String DB_URL = "jdbc:h2:" + - System.getProperty("user.dir") + File.separator + "KeepAlive" + File.separator + "keppalive"; - private static final String USER = "sa"; - private static final String PASS = ""; - - static { - try { - Class.forName(JDBC_DRIVER); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); - } - } - - public static Connection getConnection() { - try { - return DriverManager.getConnection(DB_URL, USER, PASS); - } catch (SQLException e) { - throw new RuntimeException(e); - } - } -} diff --git a/src/keepalive/repository/IDatabaseBlock.java b/src/keepalive/repository/IDatabaseBlock.java new file mode 100644 index 0000000..13b36df --- /dev/null +++ b/src/keepalive/repository/IDatabaseBlock.java @@ -0,0 +1,17 @@ +package keepalive.repository; + +import freenet.keys.FreenetURI; + +public interface IDatabaseBlock { + + int MAX_SIZE = 32768; + + FreenetURI getUri(); + + void setUri(FreenetURI uri); + + byte[] getData(); + + void setData(byte[] data); + +} diff --git a/src/keepalive/repository/IDatabaseDAO.java b/src/keepalive/repository/IDatabaseDAO.java new file mode 100644 index 0000000..c9f20db --- /dev/null +++ b/src/keepalive/repository/IDatabaseDAO.java @@ -0,0 +1,53 @@ +/* + * Keep Alive Plugin + * Copyright (C) 2012 Jeriadoc + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package keepalive.repository; + +import freenet.keys.FreenetURI; +import keepalive.exceptions.DAOException; + +/** + * Interface for a database + */ +public interface IDatabaseDAO { + + /** + * this method gets called then the plugin starts + */ + default void pluginStart() {} + + /** + * this method gets called then the plugin terminates + */ + default void pluginTerminate() {} + + IDatabaseBlock create(FreenetURI uri, byte[] data) throws DAOException; + + IDatabaseBlock read(FreenetURI uri) throws DAOException; + + void update(IDatabaseBlock databaseBlock) throws DAOException; + + void delete(FreenetURI uri) throws DAOException; + + boolean exist(FreenetURI uri) throws DAOException; + + long lastAccessDiff(FreenetURI uri) throws DAOException; + + void lastAccessUpdate(FreenetURI uri) throws DAOException; + +} diff --git a/src/keepalive/repository/impl/DatabaseBlock.java b/src/keepalive/repository/impl/DatabaseBlock.java new file mode 100644 index 0000000..e036d80 --- /dev/null +++ b/src/keepalive/repository/impl/DatabaseBlock.java @@ -0,0 +1,62 @@ +package keepalive.repository.impl; + +import java.util.Arrays; +import java.util.Objects; + +import freenet.keys.FreenetURI; +import keepalive.repository.IDatabaseBlock; + +public class DatabaseBlock implements IDatabaseBlock { + + private FreenetURI uri; + private byte[] data; + + public DatabaseBlock(FreenetURI uri, byte[] data) { + this.uri = uri; + this.data = data; + } + + @Override + public FreenetURI getUri() { + return uri; + } + + @Override + public void setUri(FreenetURI uri) { + this.uri = uri; + } + + @Override + public byte[] getData() { + return data; + } + + @Override + public void setData(byte[] data) { + this.data = data; + } + + @Override + public String toString() { + return String.format("DatabaseBlock [uri=%s, data=%s]", uri, Arrays.toString(data)); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + Arrays.hashCode(data); + return prime * result + Objects.hash(uri); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if ((obj == null) || (getClass() != obj.getClass())) + return false; + final DatabaseBlock other = (DatabaseBlock) obj; + return Arrays.equals(data, other.data) && Objects.equals(uri, other.uri); + } + +} diff --git a/src/keepalive/repository/impl/H2DatabaseDAO.java b/src/keepalive/repository/impl/H2DatabaseDAO.java new file mode 100644 index 0000000..77496c3 --- /dev/null +++ b/src/keepalive/repository/impl/H2DatabaseDAO.java @@ -0,0 +1,251 @@ +/* + * Keep Alive Plugin + * Copyright (C) 2012 Jeriadoc + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +package keepalive.repository.impl; + +import java.io.File; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Properties; + +import org.h2.jdbc.JdbcSQLNonTransientException; +import org.h2.tools.Upgrade; + +import freenet.keys.FreenetURI; +import keepalive.Plugin; +import keepalive.exceptions.DAOException; +import keepalive.exceptions.DatabaseException; +import keepalive.model.PropertiesKey; +import keepalive.repository.IDatabaseBlock; +import keepalive.repository.IDatabaseDAO; + +/** + * The H2 database settings + */ +public class H2DatabaseDAO implements IDatabaseDAO { + + private final Plugin plugin; + + private static final String JDBC_DRIVER = "org.h2.Driver"; + private static final String DB_URL = String.format("jdbc:h2:%s%sKeepAlive%2$skeppalive", System.getProperty("user.dir"), File.separator); + private static final String USER = "sa"; + private static final String PASS = ""; + private static int updateTries = 0; + private static Class jdbcDriverClass = null; + + private static final String SQL_SAVE = "INSERT INTO Block (uri, data) VALUES (?, ?)"; + private static final String SQL_FIND = "SELECT data FROM Block WHERE uri = ?"; + private static final String SQL_UPDATE = "UPDATE Block SET data = ? WHERE uri = ?"; + private static final String SQL_DELETE = "DELETE FROM Block WHERE uri = ?;"; + private static final String SQL_LAST_ACCESS_DIFF = "SELECT TIMESTAMPDIFF(MILLISECOND, last_access, CURRENT_TIMESTAMP) FROM Block WHERE uri = ?"; + private static final String SQL_LAST_ACCESS_UPDATE = "UPDATE Block SET last_access = CURRENT_TIMESTAMP WHERE uri = ?"; + + public H2DatabaseDAO(Plugin plugin) { + this.plugin = plugin; + } + + public synchronized Connection getConnection() throws DatabaseException { + try { + if (jdbcDriverClass == null) + jdbcDriverClass = Class.forName(JDBC_DRIVER); + + return DriverManager.getConnection(DB_URL, USER, PASS); + } catch (final JdbcSQLNonTransientException e) { + // try one time to upgrade the database + if (updateTries++ <= 0 && e.getMessage().contains("The write format 1 is smaller than the supported format 2")) { + upgradeDB(); + return getConnection(); + } + + throw new DatabaseException(e); + } catch (final Exception e) { + throw new DatabaseException(e); + } + } + + @Override + public void pluginStart() { + try (Connection connection = getConnection(); + Statement statement = connection.createStatement()) { + final String sql = String.format("CREATE TABLE IF NOT EXISTS Block (" + + "uri VARCHAR(256) PRIMARY KEY, " + + "data VARBINARY(%s) not null, " + + "last_access TIMESTAMP DEFAULT CURRENT_TIMESTAMP)", IDatabaseBlock.MAX_SIZE); + statement.executeUpdate(sql); + } catch (final Exception e) { + plugin.log(e.getMessage(), e); + } + } + + private void upgradeDB() throws DatabaseException { + try { + final int fromDBVersion = plugin.getIntProp(PropertiesKey.DB_VERSION); + plugin.log("upgradeDB from version -> " + fromDBVersion); + + final Properties props = new Properties(); + props.put("user", USER); + props.put("password", PASS); + final boolean updateBool = Upgrade.upgrade(DB_URL, props, fromDBVersion); + if (updateBool) { + plugin.setProp(PropertiesKey.DB_VERSION, "206"); + plugin.saveProp(); + plugin.log("upgradeDB successful to version -> " + plugin.getIntProp(PropertiesKey.DB_VERSION)); + } else { + plugin.log("upgradeDB was NOT successful"); + } + } catch (final Exception e) { + throw new DatabaseException(e); + } + } + + @Override + public IDatabaseBlock create(FreenetURI uri, byte[] data) throws DAOException { + if (exist(uri)) + throw new DAOException("The block uri: '%s' is already saved", uri); + if (data == null) + throw new DAOException("The data need to be not null for block uri: %s", uri); + if (data.length > IDatabaseBlock.MAX_SIZE) + throw new DAOException("The data is bigger (size: %s) as allowed for block uri: %s", data.length, uri); + + IDatabaseBlock result = null; + + try (Connection connection = getConnection(); + PreparedStatement savePreparedStatement = connection.prepareStatement(SQL_SAVE)) { + savePreparedStatement.setString(1, uri.toString()); + savePreparedStatement.setBytes(2, data); + savePreparedStatement.executeUpdate(); + + result = new DatabaseBlock(uri, data); + } catch (SQLException | DatabaseException e) { + throw new DAOException("DatabaseBlock for uri: %s couldnt be created", e, uri); + } + + return result; + } + + @Override + public IDatabaseBlock read(FreenetURI uri) throws DAOException { + if (uri == null) + throw new DAOException("The uri need to be not null!"); + + IDatabaseBlock result = null; + try (Connection connection = getConnection(); + PreparedStatement preparedStatement = connection.prepareStatement(SQL_FIND)) { + preparedStatement.setString(1, uri.toString()); + + final ResultSet resultSet = preparedStatement.executeQuery(); + if (resultSet.next()) { + final byte[] data = resultSet.getBytes("data"); + + if (resultSet.next()) + throw new DAOException("DatabaseBlock for uri: %s is not unique", uri); + + result = new DatabaseBlock(uri, data); + } + } catch (SQLException | DatabaseException e) { + throw new DAOException("DatabaseBlock for uri: %s couldnt be loaded", e, uri); + } + + return result; + } + + @Override + public void update(IDatabaseBlock databaseBlock) throws DAOException { + if (databaseBlock == null) + throw new DAOException("The databaseBlock need to be not null!"); + if (!exist(databaseBlock.getUri())) + throw new DAOException("The databaseBlock uri: '%s' doesnt exist", databaseBlock.getUri()); + + try (Connection connection = getConnection(); + PreparedStatement updatePreparedStatement = connection.prepareStatement(SQL_UPDATE)) { + updatePreparedStatement.setBytes(1, databaseBlock.getData()); + updatePreparedStatement.setString(2, databaseBlock.getUri().toString()); + updatePreparedStatement.executeUpdate(); + } catch (SQLException | DatabaseException e) { + throw new DAOException("DatabaseBlock for uri: %s couldnt be updated", e, databaseBlock.getUri()); + } + } + + @Override + public void delete(FreenetURI uri) throws DAOException { + if (!exist(uri)) + return; + + try (Connection connection = getConnection(); + PreparedStatement preparedStatement = connection.prepareStatement(SQL_DELETE)) { + preparedStatement.setString(1, uri.toString()); + preparedStatement.executeUpdate(); + } catch (SQLException | DatabaseException e) { + throw new DAOException("DatabaseBlock for uri: %s couldnt be deleted", e, uri); + } + } + + @Override + public boolean exist(FreenetURI uri) throws DAOException { + if (uri == null) + throw new DAOException("The uri need to be not null!"); + + return read(uri) != null; + } + + @Override + public long lastAccessDiff(FreenetURI uri) throws DAOException { + if (uri == null) + throw new DAOException("The uri need to be not null!"); + + try (Connection connection = getConnection(); + PreparedStatement preparedStatement = connection.prepareStatement(SQL_LAST_ACCESS_DIFF)) { + preparedStatement.setString(1, uri.toString()); + final ResultSet resultSet = preparedStatement.executeQuery(); + if (!resultSet.next()) { + return 0; + } + final long diff = resultSet.getLong(1); + + if (resultSet.next()) { + plugin.log("Not unique uri: " + uri); + return 0; + } + + return diff; + } catch (final Exception e) { + plugin.log("%s %s", e, e.getMessage(), uri); + } + + return 0; + } + + @Override + public void lastAccessUpdate(FreenetURI uri) throws DAOException { + if (uri == null) + throw new DAOException("The uri need to be not null!"); + + try (Connection connection = getConnection(); + PreparedStatement preparedStatement = connection.prepareStatement(SQL_LAST_ACCESS_UPDATE)) { + preparedStatement.setString(1, uri.toString()); + preparedStatement.executeUpdate(); + } catch (final Exception e) { + throw new DAOException("lastAccessUpdate error", e); + } + } + +} diff --git a/src/keepalive/service/net/Client.java b/src/keepalive/service/net/Client.java index 9dbf3ac..a1d966e 100644 --- a/src/keepalive/service/net/Client.java +++ b/src/keepalive/service/net/Client.java @@ -1,52 +1,63 @@ package keepalive.service.net; -import freenet.client.*; +import freenet.client.FetchContext; +import freenet.client.FetchException; +import freenet.client.FetchResult; +import freenet.client.FetchWaiter; +import freenet.client.HighLevelSimpleClientImpl; +import freenet.client.InsertBlock; +import freenet.client.InsertException; import freenet.keys.FreenetURI; import freenet.node.RequestClient; import freenet.support.io.ArrayBucket; public class Client { - - private static RequestClient rc = new RequestClient() { - - @Override - public boolean persistent() { - return false; - } - - @Override - public boolean realTimeFlag() { - return true; - } - }; - - // fetch raw data - public static FetchResult fetch(FreenetURI uri, HighLevelSimpleClientImpl hlsc) throws FetchException { - uri = normalizeUri(uri); - assert uri != null; - if (uri.isCHK()) { - uri.getExtra()[2] = 0; // deactivate control flag - } - - FetchContext fetchContext = hlsc.getFetchContext(); - fetchContext.returnZIPManifests = true; - FetchWaiter fetchWaiter = new FetchWaiter(rc); - hlsc.fetch(uri, -1, fetchWaiter, fetchContext); - return fetchWaiter.waitForCompletion(); - } - - public static FreenetURI insert(FreenetURI uri, byte[] data, HighLevelSimpleClientImpl hlsc) throws InsertException { - InsertBlock insert = new InsertBlock(new ArrayBucket(data), null, uri); - return hlsc.insert(insert, false, null); - } - - public static FreenetURI normalizeUri(FreenetURI uri) { - if (uri.isUSK()) { - uri = uri.sskForUSK(); - } - if (uri.hasMetaStrings()) { - uri = uri.setMetaString(null); - } - return uri; - } + + private static RequestClient rc = new RequestClient() { + + @Override + public boolean persistent() { + return false; + } + + @Override + public boolean realTimeFlag() { + return true; + } + + }; + + private Client() {} + + // fetch raw data + public static FetchResult fetch(FreenetURI uri, HighLevelSimpleClientImpl hlsc) throws FetchException { + uri = normalizeUri(uri); + if (uri == null) + return null; + + if (uri.isCHK()) + uri.getExtra()[2] = 0; // deactivate control flag + + final FetchContext fetchContext = hlsc.getFetchContext(); + fetchContext.returnZIPManifests = true; + final FetchWaiter fetchWaiter = new FetchWaiter(rc); + hlsc.fetch(uri, Long.MAX_VALUE, fetchWaiter, fetchContext); // TODO/FIXME: after Fred update (>1495) Long.MAX_VALUE to -1 again + return fetchWaiter.waitForCompletion(); + } + + public static FreenetURI insert(FreenetURI uri, byte[] data, HighLevelSimpleClientImpl hlsc) throws InsertException { + final InsertBlock insert = new InsertBlock(new ArrayBucket(data), null, uri); + return hlsc.insert(insert, false, null); + } + + public static FreenetURI normalizeUri(FreenetURI uri) { + if (uri.isUSK()) + uri = uri.sskForUSK(); + + if (uri.hasMetaStrings()) + uri = uri.setMetaString(null); + + return uri; + } + } diff --git a/src/keepalive/service/net/FetchFailedException.java b/src/keepalive/service/net/FetchFailedException.java deleted file mode 100644 index 5db3160..0000000 --- a/src/keepalive/service/net/FetchFailedException.java +++ /dev/null @@ -1,8 +0,0 @@ -package keepalive.service.net; - -public class FetchFailedException extends Exception { - - public FetchFailedException(String message) { - super(message); - } -} diff --git a/src/keepalive/service/net/HLSCIgnoreStore.java b/src/keepalive/service/net/HLSCIgnoreStore.java index 3d96063..8529d9b 100644 --- a/src/keepalive/service/net/HLSCIgnoreStore.java +++ b/src/keepalive/service/net/HLSCIgnoreStore.java @@ -4,30 +4,31 @@ import freenet.client.HighLevelSimpleClientImpl; public class HLSCIgnoreStore extends HighLevelSimpleClientImpl { - - private static volatile HLSCIgnoreStore hlscIgnoreStore; - - private HLSCIgnoreStore(HighLevelSimpleClientImpl hlsc) { - super(hlsc); - } - - static HLSCIgnoreStore getInstance(HighLevelSimpleClientImpl hlsc) { - HLSCIgnoreStore localHlscIgnoreStore = hlscIgnoreStore; - if (localHlscIgnoreStore == null) { - synchronized (HLSCIgnoreStore.class) { - localHlscIgnoreStore = hlscIgnoreStore; - if (localHlscIgnoreStore == null) { - hlscIgnoreStore = localHlscIgnoreStore = new HLSCIgnoreStore(hlsc); - } - } - } - return localHlscIgnoreStore; - } - - @Override - public FetchContext getFetchContext() { - FetchContext fc = super.getFetchContext(); - fc.ignoreStore = true; - return fc; - } + + private static volatile HLSCIgnoreStore hlscIgnoreStore; + + private HLSCIgnoreStore(HighLevelSimpleClientImpl hlsc) { + super(hlsc); + } + + static HLSCIgnoreStore getInstance(HighLevelSimpleClientImpl hlsc) { + HLSCIgnoreStore localHlscIgnoreStore = hlscIgnoreStore; + if (localHlscIgnoreStore == null) { + synchronized (HLSCIgnoreStore.class) { + localHlscIgnoreStore = hlscIgnoreStore; + if (localHlscIgnoreStore == null) { + hlscIgnoreStore = localHlscIgnoreStore = new HLSCIgnoreStore(hlsc); + } + } + } + return localHlscIgnoreStore; + } + + @Override + public FetchContext getFetchContext() { + final FetchContext fc = super.getFetchContext(); + fc.ignoreStore = true; + return fc; + } + } diff --git a/src/keepalive/service/net/SingleFetch.java b/src/keepalive/service/net/SingleFetch.java index 51a5da1..4cc6bcb 100644 --- a/src/keepalive/service/net/SingleFetch.java +++ b/src/keepalive/service/net/SingleFetch.java @@ -18,83 +18,99 @@ */ package keepalive.service.net; +import java.io.IOException; +import java.util.concurrent.Callable; + import freenet.client.FetchException; import freenet.client.FetchResult; import freenet.keys.FreenetURI; import freenet.support.io.ArrayBucket; +import keepalive.Plugin; +import keepalive.model.IBlock; +import keepalive.service.reinserter.FetchBlocksResult; import keepalive.service.reinserter.Reinserter; -import keepalive.model.Block; - -import java.io.IOException; -import java.util.concurrent.Callable; public class SingleFetch extends SingleJob implements Callable { - - private final boolean persistenceCheck; - - public SingleFetch(Reinserter reinserter, Block block, boolean persistenceCheck) { - super(reinserter, "fetch", block); - - this.persistenceCheck = persistenceCheck; - } - - @Override - public Boolean call() { - Thread.currentThread().setName("KeepAlive SingleFetch"); - FetchResult fetchResult = null; - boolean fetchSuccessful = false; - - try { - - // init - HLSCIgnoreStore hlscIgnoreStore = HLSCIgnoreStore.getInstance(plugin.getFreenetClient()); - - FreenetURI fetchUri = getUri(); - block.setFetchDone(false); - block.setFetchSuccessful(false); - - // request - try { - - if (!persistenceCheck) { - fetchResult = plugin.getFreenetClient().fetch(fetchUri); - } else { - fetchResult = hlscIgnoreStore.fetch(fetchUri); - } - - } catch (FetchException e) { - block.setResultLog("-> fetch error: " + e.getMessage()); - } - - if (Thread.currentThread().isInterrupted()) { - return false; - } - - // log / success flag - if (block.getResultLog() == null) { - if (fetchResult == null) { - block.setResultLog("-> fetch failed"); - } else { - block.setBucket(new ArrayBucket(fetchResult.asByteArray())); - block.setFetchSuccessful(true); - block.setResultLog("-> fetch successful"); - fetchSuccessful = true; - } - } - - //finish - reinserter.registerBlockFetchSuccess(block); - block.setFetchDone(true); - - } catch (IOException e) { - log("SingleFetch.run(): " + e.getMessage(), 0); - } finally { - if (fetchResult != null && fetchResult.asBucket() != null) { - fetchResult.asBucket().free(); - } - finish(); - } - - return fetchSuccessful; - } + + private boolean persistenceCheck; + private FetchBlocksResult fetchBlocksResult; + + public SingleFetch(Reinserter reinserter, IBlock block) { + super(reinserter, "fetch", block); + + this.persistenceCheck = false; + } + + public SingleFetch(Reinserter reinserter, IBlock block, FetchBlocksResult fetchBlocksResult) { + this(reinserter, block); + + this.persistenceCheck = true; + this.fetchBlocksResult = fetchBlocksResult; + } + + @Override + public Boolean call() { + final Boolean result = fetch(); + + if (fetchBlocksResult != null) + fetchBlocksResult.addResult(result); + + return result; + } + + public Boolean fetch() { + Thread.currentThread().setName(Plugin.PLUGIN_NAME + " SingleFetch"); + FetchResult fetchResult = null; + boolean fetchSuccessful = false; + + try { + // init + final HLSCIgnoreStore hlscIgnoreStore = HLSCIgnoreStore.getInstance(plugin.getFreenetClient()); + + final FreenetURI fetchUri = getUri(); + block.setFetchDone(false); + block.setFetchSuccessful(false); + + // request + try { + if (!persistenceCheck) { + fetchResult = plugin.getFreenetClient().fetch(fetchUri); + } else { + fetchResult = hlscIgnoreStore.fetch(fetchUri); + } + } catch (final FetchException e) { + block.setResultLog("error: " + e.getMessage()); + } + + if (Thread.currentThread().isInterrupted()) { + return false; + } + + // log / success flag + if (block.getResultLog() == null) { + if (fetchResult == null) { + block.setResultLog("failed"); + } else { + block.setBucket(new ArrayBucket(fetchResult.asByteArray())); + block.setFetchSuccessful(true); + block.setResultLog("successful"); + fetchSuccessful = true; + } + } + + //finish + reinserter.registerBlockFetchSuccess(block); + block.setFetchDone(true); + } catch (final IOException e) { + log("SingleFetch.run(): " + e.getMessage(), 0); + } finally { + if (fetchResult != null && fetchResult.asBucket() != null) { + fetchResult.asBucket().free(); + } + finish(); + } + + return fetchSuccessful; + } + } diff --git a/src/keepalive/service/net/SingleInsert.java b/src/keepalive/service/net/SingleInsert.java index 0b1e9d3..6faae0b 100644 --- a/src/keepalive/service/net/SingleInsert.java +++ b/src/keepalive/service/net/SingleInsert.java @@ -22,106 +22,106 @@ import freenet.client.InsertContext; import freenet.client.InsertException; import freenet.keys.FreenetURI; -import keepalive.service.reinserter.Reinserter; -import keepalive.model.Block; +import keepalive.Plugin; +import keepalive.model.IBlock; import keepalive.model.Segment; +import keepalive.service.reinserter.Reinserter; public class SingleInsert extends SingleJob implements Runnable { - - public SingleInsert(Reinserter reinserter, Block block) { - super(reinserter, "insertion", block); - } - - @Override - public String toString() { - return "KeepAlive - SingleInsert"; - } - - @Override - public void run() { - Thread.currentThread().setName("KeepAlive SingleInsert"); - - FreenetURI fetchUri = getUri(); - block.setInsertDone(false); - block.setInsertSuccessful(false); - - try { - - // fetch - if (block.getBucket() == null) { - SingleFetch singleFetch = new SingleFetch(reinserter, block, false); - singleFetch.call(); - if (!reinserter.isActive()) { - return; - } - } - - Segment segment = reinserter.getSegments().get(block.getSegmentId()); - - if (block.getBucket() == null) { - block.setResultLog("-> insertion failed: fetch failed"); - } else { // insert - if (Thread.currentThread().isInterrupted()) { - return; - } - - try { - - InsertBlock insertBlock = new InsertBlock(block.getBucket(), null, fetchUri); - InsertContext insertContext = plugin.getFreenetClient().getInsertContext(true); - - if (compressionAlgorithm != null && !compressionAlgorithm.equals("none")) { - insertContext.compressorDescriptor = compressionAlgorithm; - } - - // switch to crypto_algorithm 2 (instead of using the new one that is introduced since 1416) - if (uriExtra[1] == 2) { - insertContext.setCompatibilityMode(InsertContext.CompatibilityMode.COMPAT_1255); - } - - // don't triple-insert blocks. - insertContext.extraInsertsSingleBlock = 0; - insertContext.earlyEncode = false; - - // re-insert top blocks and single key files at very high priority, all others at medium prio. - short prio = segment.size() == 1 ? (short) 1 : (short) 3; - - FreenetURI insertUri = plugin.getFreenetClient() - .insert(insertBlock, null, false, prio, insertContext, fetchUri.getCryptoKey()); - - // insert finished - if (!reinserter.isActive()) { - return; - } - - if (insertUri != null) { - if (fetchUri.equals(insertUri)) { - block.setInsertSuccessful(true); - block.setResultLog("-> inserted: " + insertUri.toString()); - } else { - block.setResultLog("-> insertion failed - different uri: " + insertUri.toString()); - } - } else { - block.setResultLog("-> insertion failed"); - } - - } catch (InsertException e) { - block.setResultLog("-> insertion error: " + e.getMessage()); - } - } - - // reg success if single-block-segment - if (segment.size() == 1) { - reinserter.updateSegmentStatistic(segment, block.isInsertSuccessful()); - } - - // finish - block.setInsertDone(true); - - } catch (Exception e) { - log("SingleInsert.run(): " + e.getMessage(), 0); - } finally { - finish(); - } - } + + public SingleInsert(Reinserter reinserter, IBlock block) { + super(reinserter, "insertion", block); + } + + @Override + public String toString() { + return Plugin.PLUGIN_NAME + " - SingleInsert"; + } + + @Override + public void run() { + Thread.currentThread().setName(Plugin.PLUGIN_NAME + " SingleInsert"); + + final FreenetURI fetchUri = getUri(); + block.setInsertDone(false); + block.setInsertSuccessful(false); + + try { + // fetch + if (block.getBucket() == null) { + final SingleFetch singleFetch = new SingleFetch(reinserter, block); + singleFetch.call(); + if (!reinserter.isActive()) { + return; + } + } + + final Segment segment = reinserter.getSegments().get(block.getSegmentId()); + + if (block.getBucket() == null) { + block.setResultLog("failed: fetch failed"); + } else { // insert + if (Thread.currentThread().isInterrupted()) { + return; + } + + try { + final InsertBlock insertBlock = new InsertBlock(block.getBucket(), null, fetchUri); + final InsertContext insertContext = plugin.getFreenetClient().getInsertContext(true); + + if (compressionAlgorithm != null && !"none".equals(compressionAlgorithm)) { + insertContext.compressorDescriptor = compressionAlgorithm; + } + + // switch to crypto_algorithm 2 (instead of using the new one that is introduced since 1416) + if (uriExtra[1] == 2) { + insertContext.setCompatibilityMode(InsertContext.CompatibilityMode.COMPAT_1255); + } + + // don't triple-insert blocks. + insertContext.extraInsertsSingleBlock = 0; + insertContext.earlyEncode = false; + + // re-insert top blocks and single key files at very high priority, all others at medium prio. + final short prio = segment.size() == 1 ? (short) 1 : (short) 3; + + final FreenetURI insertUri = plugin.getFreenetClient() + .insert(insertBlock, null, false, prio, insertContext, fetchUri.getCryptoKey()); + + // insert finished + if (!reinserter.isActive()) { + return; + } + + if (insertUri != null) { + if (fetchUri.equals(insertUri)) { + block.setInsertSuccessful(true); + block.setResultLog("inserted: " + insertUri.toString()); + } else { + block.setResultLog("failed - different uri: " + insertUri.toString()); + } + } else { + block.setResultLog("failed"); + } + + } catch (final InsertException e) { + block.setResultLog("error: " + e.getMessage()); + } + } + + // reg success if single-block-segment + if (segment.size() == 1) { + reinserter.updateSegmentStatistic(segment, block.isInsertSuccessful()); + } + + // finish + block.setInsertDone(true); + + } catch (final Exception e) { + log("SingleInsert.run(): " + e.getMessage(), 0); + } finally { + finish(); + } + } + } diff --git a/src/keepalive/service/net/SingleJob.java b/src/keepalive/service/net/SingleJob.java index 14f13ec..941a1dd 100644 --- a/src/keepalive/service/net/SingleJob.java +++ b/src/keepalive/service/net/SingleJob.java @@ -21,71 +21,67 @@ import freenet.keys.FreenetURI; import freenet.support.compress.Compressor; import keepalive.Plugin; +import keepalive.model.IBlock; import keepalive.service.reinserter.Reinserter; -import keepalive.model.Block; public abstract class SingleJob { - - public static final int MAX_LIFETIME = 30; - - Plugin plugin; - Reinserter reinserter; - Block block; - byte[] uriExtra; - String compressionAlgorithm; - - private String jobType; - - SingleJob(Reinserter reinserter, String jobType, Block block) { - this.reinserter = reinserter; - this.jobType = jobType; - this.block = block; - this.plugin = reinserter.getPlugin(); - } - - FreenetURI getUri() { - FreenetURI uri = block.getUri().clone(); - - // modify the control flag of the URI to get always the raw data - uriExtra = uri.getExtra(); - uriExtra[2] = 0; - - // get the compression algorithm of the block - if (uriExtra[4] >= 0) { - compressionAlgorithm = - Compressor.COMPRESSOR_TYPE.getCompressorByMetadataID(uriExtra[4]).name; - } else { - compressionAlgorithm = "none"; - } - - log("request: " + block.getUri().toString() + - " (crypt=" + uriExtra[1] + - ",control=" + block.getUri().getExtra()[2] + - ",compress=" + uriExtra[4] + "=" + compressionAlgorithm + ")", 2); - - return uri; - } - - void finish() { - if (reinserter.isActive() && !reinserter.isInterrupted()) { - // log - String firstLog = jobType + ": " + block.getUri(); - if (!block.isFetchSuccessful() && !block.isInsertSuccessful()) { - firstLog = "" + firstLog + ""; - block.setResultLog("" + block.getResultLog() + ""); - } - log(firstLog); - log(block.getResultLog()); - } - } - - protected void log(String message, int logLevel) { - if (reinserter.isActive() && !Thread.currentThread().isInterrupted() && !reinserter.isInterrupted()) { - reinserter.log(block.getSegmentId(), message, 0, logLevel); - } - } - - protected void log(String message) { - log(message, 1); - } + + public static final int MAX_LIFETIME = 30; + + protected Plugin plugin; + protected Reinserter reinserter; + protected IBlock block; + protected byte[] uriExtra; + protected String compressionAlgorithm; + + private final String jobType; + + protected SingleJob(Reinserter reinserter, String jobType, IBlock block) { + this.reinserter = reinserter; + this.jobType = jobType; + this.block = block; + this.plugin = reinserter.getPlugin(); + } + + protected FreenetURI getUri() { + final FreenetURI uri = block.getUri().clone(); + + // modify the control flag of the URI to get always the raw data + uriExtra = uri.getExtra(); + uriExtra[2] = 0; + + // get the compression algorithm of the block + if (uriExtra[4] >= 0) { + compressionAlgorithm = Compressor.COMPRESSOR_TYPE.getCompressorByMetadataID(uriExtra[4]).name; + } else { + compressionAlgorithm = "none"; + } + + log(String.format("request: %s (crypt=%s,control=%s,compress=%s=%s)", block.getUri(), uriExtra[1], block.getUri().getExtra()[2], uriExtra[4], compressionAlgorithm), 2); + + return uri; + } + + protected void finish() { + if (reinserter.isActive() && !reinserter.isInterrupted()) { + String msg = String.format("%s: %s -> %s", jobType, block.getResultLog(), block.getUri()); + + // error or problem + if (!block.isFetchSuccessful() && !block.isInsertSuccessful()) + msg = String.format("%s", msg); + + log(msg); + } + } + + protected void log(String message, int logLevel) { + if (reinserter.isActive() && !Thread.currentThread().isInterrupted() && !reinserter.isInterrupted()) { + reinserter.log(block.getSegmentId(), message, 0, logLevel); + } + } + + protected void log(String message) { + log(message, 1); + } + } diff --git a/src/keepalive/service/reinserter/FetchBlocksResult.java b/src/keepalive/service/reinserter/FetchBlocksResult.java new file mode 100644 index 0000000..1c85ecf --- /dev/null +++ b/src/keepalive/service/reinserter/FetchBlocksResult.java @@ -0,0 +1,20 @@ +package keepalive.service.reinserter; + +public class FetchBlocksResult { + + private int successful = 0; + private int failed = 0; + + public synchronized void addResult(boolean successful) { + if (successful) { + this.successful++; + } else { + failed++; + } + } + + public double calculatePersistenceRate() { + return (double) successful / (successful + failed); + } + +} diff --git a/src/keepalive/service/reinserter/Reinserter.java b/src/keepalive/service/reinserter/Reinserter.java index 1594212..01bbb2e 100644 --- a/src/keepalive/service/reinserter/Reinserter.java +++ b/src/keepalive/service/reinserter/Reinserter.java @@ -18,1215 +18,1057 @@ */ package keepalive.service.reinserter; -import freenet.client.*; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.zip.ZipInputStream; + +import org.apache.tools.tar.TarInputStream; + import freenet.client.ArchiveManager.ARCHIVE_TYPE; -import freenet.client.InsertContext.CompatibilityMode; +import freenet.client.FECCodec; +import freenet.client.FetchContext; +import freenet.client.FetchException; +import freenet.client.FetchResult; +import freenet.client.FetchWaiter; +import freenet.client.InsertException; +import freenet.client.Metadata; import freenet.client.Metadata.SplitfileAlgorithm; -import freenet.client.async.ClientBaseCallback; +import freenet.client.MetadataParseException; import freenet.client.async.ClientContext; -import freenet.client.async.ClientGetState; -import freenet.client.async.ClientRequestSchedulerGroup; -import freenet.client.async.ClientRequester; -import freenet.client.async.GetCompletionCallback; import freenet.client.async.SplitFileFetcher; import freenet.client.async.SplitFileSegmentKeys; -import freenet.client.async.StreamGenerator; -import freenet.crypt.HashResult; import freenet.keys.CHKBlock; import freenet.keys.FreenetURI; -import freenet.node.RequestClient; import freenet.pluginmanager.PluginRespirator; -import freenet.support.compress.Compressor; import freenet.support.compress.Compressor.COMPRESSOR_TYPE; import freenet.support.io.ArrayBucket; - -import java.io.*; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map.Entry; -import java.util.concurrent.*; -import java.util.zip.ZipInputStream; - import keepalive.Plugin; +import keepalive.exceptions.DAOException; +import keepalive.exceptions.DuplicateKeyException; +import keepalive.exceptions.FetchFailedException; import keepalive.model.Block; +import keepalive.model.IBlock; +import keepalive.model.PropertiesKey; import keepalive.model.Segment; -import keepalive.repository.BlockRepository; -import keepalive.service.net.*; -import org.apache.tools.tar.TarInputStream; +import keepalive.repository.IDatabaseBlock; +import keepalive.service.net.Client; +import keepalive.service.net.SingleFetch; +import keepalive.service.net.SingleInsert; +import keepalive.service.net.SingleJob; +import keepalive.urivalues.IUriValue; public final class Reinserter extends Thread { - - private final Plugin plugin; - private final int siteId; - private final CountDownLatch latch; - private PluginRespirator pr; - private long lastActivityTime; - private HashMap manifestURIs; - private HashMap blocks; - private int parsedSegmentId; - private int parsedBlockId; - private ArrayList segments = new ArrayList<>(); - - public Reinserter(Plugin plugin, int siteId, CountDownLatch latch) { - this.plugin = plugin; - this.siteId = siteId; - this.latch = latch; - this.setName("KeepAlive ReInserter " + siteId); - } - - @Override - public void run() { - try { - - // init - pr = plugin.pluginContext.pluginRespirator; - manifestURIs = new HashMap<>(); - blocks = new HashMap<>(); - String uriProp = plugin.getProp("uri_" + siteId); - plugin.log("start reinserter for site " + uriProp + " (" + siteId + ")", 1); - plugin.clearLog(plugin.getLogFilename(siteId)); - isActive(true); - long startedAt = System.currentTimeMillis(); - long timeLeft = TimeUnit.HOURS.toMillis(plugin.getIntProp("single_url_timeslot")); - - FreenetURI uri = new FreenetURI(uriProp); - - // update if USK - if (uri.isUSK()) { - FreenetURI newUri = updateUsk(uri); - if (newUri != null && !newUri.equals(uri)) { - String newUriString = newUri.toString(); - plugin.log("received new uri: " + newUriString, 1); - if (plugin.isDuplicate(newUriString)) { - plugin.log("remove uri as duplicate: " + newUriString, 1); - plugin.removeUri(siteId); - return; - } else { - plugin.setProp("uri_" + siteId, newUri.toString()); - plugin.setProp("blocks_" + siteId, "?"); - uri = newUri; - } - } - } - - // check top block availability - BlockRepository blockRepository = BlockRepository.getInstance(plugin); - FreenetURI topBlockUri = Client.normalizeUri(uri.clone()); - if (blockRepository.lastAccessDiff(topBlockUri.toString()) > TimeUnit.DAYS.toMillis(1)) { - try { - Client.fetch(topBlockUri, plugin.getFreenetClient()); - } catch (FetchException e) { - log(e.getShortMessage(), 0, 0); - try { - FreenetURI insertUri = Client.insert( - topBlockUri, blockRepository.findOne(topBlockUri.toString()), plugin.getFreenetClient()); - - if (insertUri != null) { - if (topBlockUri.equals(insertUri)) { - log("Successfully inserted top block: " + insertUri.toString(), 0); - } else { - log("Top block insertion failed - different uri: " + insertUri.toString(), 0); - } - } else { - log("Top block insertion failed (insertUri = null)", 0); - } - } catch (InsertException e1) { - log(e1.getMessage(), 0, 0); - } - } - blockRepository.lastAccessUpdate(topBlockUri.toString()); - } - - // register uri - registerManifestUri(uri, -1); - - // load list of keys (if exists) - // skip if 1 because the manifest failed to fetch before. - String numBlocks = plugin.getProp("blocks_" + siteId); - if (!numBlocks.equals("?") && !numBlocks.equals("1")) { - log("*** loading list of blocks ***", 0, 0); - loadBlockUris(); - } else { - // parse metadata - log("*** parsing data structure ***", 0, 0); - parsedSegmentId = -1; - parsedBlockId = -1; - while (manifestURIs.size() > 0) { - if (isInterrupted()) { - return; - } - - if (!isActive()) { - plugin.log("Stop after stuck state (metadata)", 0); - return; - } - - uri = (FreenetURI) manifestURIs.keySet().toArray()[0]; - log(uri.toString(), 0); - try { - parseMetadata(uri, null, 0); - } catch (FetchFailedException e) { - log(e.getMessage(), 0); - return; - } - manifestURIs.remove(uri); - } - - if (isInterrupted()) { - return; - } - - saveBlockUris(); - plugin.setIntProp("blocks_" + siteId, blocks.size()); - plugin.saveProp(); - } - - // max segment id - int maxSegmentId = -1; - for (Block block : blocks.values()) { - maxSegmentId = Math.max(maxSegmentId, block.getSegmentId()); - } - - // init reinsertion - if (plugin.getIntProp("segment_" + siteId) == maxSegmentId) { - plugin.setIntProp("segment_" + siteId, -1); - } - if (plugin.getIntProp("segment_" + siteId) == -1) { - - log("*** starting reinsertion ***", 0, 0); - - // reset success counter - StringBuilder success = new StringBuilder(); - StringBuilder segmentsSuccess = new StringBuilder(); - for (int i = 0; i <= maxSegmentId; i++) { - if (i > 0) success.append(","); - - success.append("0,0"); - segmentsSuccess.append("0"); - } - plugin.setProp("success_" + siteId, success.toString()); - plugin.setProp("success_segments_" + siteId, segmentsSuccess.toString()); - plugin.saveProp(); - - } else { - - log("*** continuing reinsertion ***", 0, 0); - - // add dummy segments - for (int i = 0; i <= plugin.getIntProp("segment_" + siteId); i++) { - segments.add(null); - } - - // reset success counter - String[] successProp = plugin.getProp("success_" + siteId).split(","); - for (int i = (plugin.getIntProp("segment_" + siteId) + 1) * 2; i < successProp.length; i++) { - successProp[i] = "0"; - } - saveSuccessToProp(successProp); - - } - - // start reinsertion - boolean doReinsertions = true; - timeLeft -= System.currentTimeMillis() - startedAt; - for (long timeSpent = 0; timeLeft - timeSpent > 0; timeSpent = System.currentTimeMillis() - startedAt, timeLeft -= timeSpent) { - startedAt = System.currentTimeMillis(); - - if (isInterrupted()) { - return; - } - - // next segment - int segmentSize = 0; - for (Block block : blocks.values()) { - if (block.getSegmentId() == segments.size()) { - segmentSize++; - } - } - if (segmentSize == 0) { - break; // ready - } - Segment segment = new Segment(this, segments.size(), segmentSize); - for (Block block : blocks.values()) { - if (block.getSegmentId() == segments.size()) { - segment.addBlock(block); - } - } - segments.add(segment); - log(segment, "*** segment size: " + segment.size(), 0); - doReinsertions = true; - - // get persistence rate of splitfile segments - if (segment.size() > 1) { - log(segment, "starting availability check for segment (n=" + - plugin.getIntProp("splitfile_test_size") + ")", 0); - - // select prove blocks - ArrayList requestedBlocks = new ArrayList<>(); - // always fetch exactly the configured number of blocks (or half segment size, whichever is smaller) - int splitfileTestSize = Math.min( - plugin.getIntProp("splitfile_test_size"), - (int) Math.ceil(segmentSize / 2.0)); - - for (int i = 0; requestedBlocks.size() < splitfileTestSize; i++) { - if (i == segmentSize) { - i = 0; - } - if ((Math.random() < (splitfileTestSize / (double) segmentSize)) - && !(requestedBlocks.contains(segment.getBlock(i)))) { - // add a block - requestedBlocks.add(segment.getBlock(i)); - } - } - - ExecutorService executor = Executors.newFixedThreadPool(plugin.getIntProp("power")); - FetchBlocksResult fetchBlocksResult = new FetchBlocksResult(); - try { - for (Block requestedBlock : requestedBlocks) { - // fetch a block - SingleFetch singleFetch = new SingleFetch(this, requestedBlock, true); - Future fetchFuture = executor.submit(singleFetch); - fetchBlocksResult.addResult(fetchFuture.get()); - } - executor.shutdown(); - boolean done = executor.awaitTermination(1, TimeUnit.HOURS); - if (!done) { - log(segment, "availability check failed", 0); - return; - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } finally { - if (!executor.isShutdown()) { - executor.shutdownNow(); - } - } - - double persistenceRate = fetchBlocksResult.calculatePersistenceRate(); - if (persistenceRate >= (double) plugin.getIntProp("splitfile_tolerance") / 100) { - doReinsertions = false; - segment.regFetchSuccess(persistenceRate); - updateSegmentStatistic(segment, true); - log(segment, "availability of segment ok: " + ((int) (persistenceRate * 100)) + - "% (approximated)", 0, 1); - checkFinishedSegments(); - if (plugin.getIntProp("segment_" + siteId) != maxSegmentId) { - log(segment, "-> segment not reinserted; moving on will resume on next pass.", 0, 1); - break; - } - } else { - log(segment, "availability of segment not ok: " + - ((int) (persistenceRate * 100)) + "% (approximated)", 0, 1); - log(segment, "-> fetch all available blocks now", 0, 1); - } - - // get all available blocks and heal the segment - if (doReinsertions) { - // add the rest of the blocks - for (int i = 0; i < segment.size(); i++) { - if (!(requestedBlocks.contains(segment.getBlock(i)))) { - // add a block - requestedBlocks.add(segment.getBlock(i)); - } - } - - executor = Executors.newFixedThreadPool(plugin.getIntProp("power")); - fetchBlocksResult = new FetchBlocksResult(); - try { - for (Block requestedBlock : requestedBlocks) { - // fetch next block that has not been fetched yet - if (requestedBlock.isFetchInProcess()) { - SingleFetch singleFetch = new SingleFetch(this, requestedBlock, true); - Future fetchFuture = executor.submit(singleFetch); - fetchBlocksResult.addResult(fetchFuture.get()); - } - } - - executor.shutdown(); - boolean done = executor.awaitTermination(1, TimeUnit.HOURS); - if (!done) { - log(segment, "get all available blocks failed", 0); - return; - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } finally { - if (!executor.isShutdown()) { - executor.shutdownNow(); - } - } - - persistenceRate = fetchBlocksResult.calculatePersistenceRate(); - if (persistenceRate >= (double) plugin.getIntProp("splitfile_tolerance") / 100.0) { - doReinsertions = false; - segment.regFetchSuccess(persistenceRate); - updateSegmentStatistic(segment, true); - log(segment, "availability of segment ok: " + ((int) (persistenceRate * 100)) + - "% (exact)", 0, 1); - checkFinishedSegments(); - if (plugin.getIntProp("segment_" + siteId) != maxSegmentId) { - log(segment, "-> segment not reinserted; moving on will resume on next pass.", 0, 1); - break; - } - } else { - log(segment, "availability of segment not ok: " + - ((int) (persistenceRate * 100)) + "% (exact)", 0, 1); - } - } - - if (doReinsertions) { // persistenceRate < splitfile tolerance - // heal segment - - // init - log(segment, "starting segment healing", 0, 1); - byte[][] dataBlocks = new byte[segment.dataSize()][]; - byte[][] checkBlocks = new byte[segment.checkSize()][]; - boolean[] dataBlocksPresent = new boolean[dataBlocks.length]; - boolean[] checkBlocksPresent = new boolean[checkBlocks.length]; - for (int i = 0; i < dataBlocks.length; i++) { - if (segment.getDataBlock(i).isFetchSuccessful()) { - dataBlocks[i] = segment.getDataBlock(i).getBucket().toByteArray(); - dataBlocksPresent[i] = true; - } else { - dataBlocks[i] = new byte[CHKBlock.DATA_LENGTH]; - dataBlocksPresent[i] = false; - } - } - for (int i = 0; i < checkBlocks.length; i++) { - if (segment.getCheckBlock(i).isFetchSuccessful()) { - checkBlocks[i] = segment.getCheckBlock(i).getBucket().toByteArray(); - checkBlocksPresent[i] = true; - } else { - checkBlocks[i] = new byte[CHKBlock.DATA_LENGTH]; - checkBlocksPresent[i] = false; - } - } - - // decode - FECCodec codec = FECCodec.getInstance(SplitfileAlgorithm.ONION_STANDARD); - log(segment, "start decoding", 0, 1); - try { - codec.decode(dataBlocks, checkBlocks, dataBlocksPresent, checkBlocksPresent, CHKBlock.DATA_LENGTH); - log(segment, "-> decoding successful", 1, 2); - } catch (Exception e) { - log(segment, "segment decoding (FEC) failed, do not reinsert", 1, 2); - updateSegmentStatistic(segment, false); - segment.setHealingNotPossible(true); - checkFinishedSegments(); - continue; - } - - // encode (= build all data blocks and check blocks from data blocks) - log(segment, "start encoding", 0, 1); - try { - codec.encode(dataBlocks, checkBlocks, checkBlocksPresent, CHKBlock.DATA_LENGTH); - log(segment, "-> encoding successful", 1, 2); - } catch (Exception e) { - log(segment, "segment encoding (FEC) failed, do not reinsert", 1, 2); - updateSegmentStatistic(segment, false); - segment.setHealingNotPossible(true); - checkFinishedSegments(); - continue; - } - - // finish - for (int i = 0; i < dataBlocks.length; i++) { - log(segment, "dataBlock_" + i, dataBlocks[i]); - segment.getDataBlock(i).setBucket(new ArrayBucket(dataBlocks[i])); - } - for (int i = 0; i < checkBlocks.length; i++) { - log(segment, "checkBlock_" + i, checkBlocks[i]); - segment.getCheckBlock(i).setBucket(new ArrayBucket(checkBlocks[i])); - } - log(segment, "segment healing (FEC) successful, start with reinsertion", 0, 1); - updateSegmentStatistic(segment, true); - } - } - - // start reinsertion - if (doReinsertions) { - - log(segment, "starting reinsertion", 0, 1); - segment.initInsert(); - - ExecutorService executor = Executors.newFixedThreadPool(plugin.getIntProp("power")); - try { - for (int i = 0; i < segment.size(); i++) { - checkFinishedSegments(); - isActive(true); - if (segment.size() > 1) { - if (segment.getBlock(i).isFetchSuccessful()) { - segment.regFetchSuccess(true); - } else { - segment.regFetchSuccess(false); - SingleInsert singleInsert = new SingleInsert(this, segment.getBlock(i)); - executor.submit(singleInsert); - } - } else { - SingleInsert singleInsert = new SingleInsert(this, segment.getBlock(i)); - executor.submit(singleInsert); - } - } - executor.shutdown(); - boolean done = executor.awaitTermination(1, TimeUnit.HOURS); - if (!done) { - log(segment, "reinsertion failed", 0); - return; - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } finally { - if (!executor.isShutdown()) { - executor.shutdownNow(); - } - } - - } - - // check if segments are finished - checkFinishedSegments(); - } - - // wait for finishing top block, if it was fetched. - if (segments.size() > 0 && segments.get(0) != null) { - while (!(segments.get(0).isFinished())) { - synchronized (this) { - try { - this.wait(1000); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - } - - if (isInterrupted()) { - return; - } - - if (!isActive()) { - plugin.log("Stop after stuck state (wait for finishing top block)", 0); - return; - } - - checkFinishedSegments(); - } - } - - // wait for finishing all segments - if (doReinsertions) { - while (plugin.getIntProp("segment_" + siteId) != maxSegmentId) { - synchronized (this) { - try { - this.wait(1_000); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - } - - if (isInterrupted()) { - return; - } - - if (!isActive()) { // TODO: this is a bypass - plugin.log("Stop after stuck state (after healing, prop segment_" + siteId + "=" + - plugin.getIntProp("segment_" + siteId) + ", maxSegmentId=" + maxSegmentId + ")", 0); - // TODO: probably segment_siteId prop should be incremented (switched to next segment) - return; - } - - checkFinishedSegments(); - } - } - - // add to history if we've processed the last segment in the file. - if (plugin.getIntProp("blocks_" + siteId) > 0 - && plugin.getIntProp("segment_" + siteId) == maxSegmentId) { - int nPersistence = (int) ((double) plugin.getSuccessValues(siteId)[0] - / plugin.getIntProp("blocks_" + siteId) * 100); - String cHistory = plugin.getProp("history_" + siteId); - String[] aHistory; - if (cHistory == null) { - aHistory = new String[]{}; - } else { - aHistory = cHistory.split(","); - } - String cThisMonth = (new SimpleDateFormat("MM.yyyy")).format(new Date()); - boolean bNewMonth = true; - if (cHistory != null && cHistory.contains(cThisMonth)) { - bNewMonth = false; - int nOldPersistence = Integer.parseInt(aHistory[aHistory.length - 1].split("-")[1]); - nPersistence = Math.min(nPersistence, nOldPersistence); - aHistory[aHistory.length - 1] = cThisMonth + "-" + nPersistence; - } - StringBuilder buf = new StringBuilder(); - for (String aHistory1 : aHistory) { - if (buf.length() > 0) { - buf.append(","); - } - buf.append(aHistory1); - } - if (bNewMonth) { - if (cHistory != null && cHistory.length() > 0) { - buf.append(","); - } - buf.append(cThisMonth).append("-").append(nPersistence); - } - cHistory = buf.toString(); - plugin.setProp("history_" + siteId, cHistory); - plugin.saveProp(); - } - - log("*** reinsertion finished ***", 0, 0); - plugin.log("reinsertion finished for " + plugin.getProp("uri_" + siteId), 1); - - } catch (Exception e) { - plugin.log("Reinserter.run()", e); - } finally { - latch.countDown(); - log("stopped", 0); - plugin.log("reinserter stopped (" + siteId + ")"); - } - } - - private void checkFinishedSegments() { - int segment; - while ((segment = plugin.getIntProp("segment_" + siteId)) < segments.size() - 1) { - if (segments.get(segment + 1).isFinished()) { - plugin.setIntProp("segment_" + siteId, segment + 1); - } else { - break; - } - } - plugin.saveProp(); - } - - private void saveBlockUris() throws IOException { - File f = new File(plugin.getPluginDirectory() + plugin.getBlockListFilename(siteId)); - if (f.exists()) { - if (!f.delete()) { - log("Reinserter.saveBlockUris(): remove block list log files was not successful.", 0); - } - } - - try (RandomAccessFile file = new RandomAccessFile(f, "rw")) { - file.setLength(0); - for (Block block : blocks.values()) { - if (file.getFilePointer() > 0) { - file.writeBytes("\n"); - } - String type = "d"; - if (!block.isDataBlock()) { - type = "c"; - } - file.writeBytes(block.getUri().toString() + "#" + block.getSegmentId() + "#" + block.getId() + "#" + type); - } - } - } - - private synchronized void loadBlockUris() throws IOException { - try (RandomAccessFile file = new RandomAccessFile( - plugin.getPluginDirectory() + plugin.getBlockListFilename(siteId), "r")) { - - String values; - while ((values = file.readLine()) != null) { - String[] aValues = values.split("#"); - FreenetURI uri = new FreenetURI(aValues[0]); - int segmentId = Integer.parseInt(aValues[1]); - int blockId = Integer.parseInt(aValues[2]); - boolean isDataBlock = aValues[3].equals("d"); - blocks.put(uri, new Block(uri, segmentId, blockId, isDataBlock)); - } - - } - } - - private void parseMetadata(FreenetURI uri, Metadata metadata, int level) - throws FetchFailedException, MetadataParseException, FetchException, IOException { - if (isInterrupted()) { - return; - } - - // register uri - registerBlockUri(uri, true, true, level); - - // constructs top level simple manifest (= first action on a new uri) - if (metadata == null) { - FetchResult fetchResult = Client.fetch(uri, plugin.getFreenetClient()); - BlockRepository.getInstance(plugin).saveOrUpdate(uri.toString(), fetchResult.asByteArray()); - - metadata = fetchManifest(uri, null, null); - if (metadata == null) { - log("no metadata", level); - return; - } - } - - // internal manifest (simple manifest) - if (metadata.isSimpleManifest()) { - log("manifest (" + getMetadataType(metadata) + "): " + metadata.getResolvedName(), level); - HashMap targetList = null; - try { - targetList = metadata.getDocuments(); - } catch (Exception ignored) { - } - - if (targetList != null) { - for (Entry entry : targetList.entrySet()) { - if (isInterrupted()) { - return; - } - // get document - Metadata target = entry.getValue(); - // remember document name - target.resolve(entry.getKey()); - // parse document - parseMetadata(uri, target, level + 1); - } - } - - return; - } - - // redirect to submanifest - if (metadata.isArchiveMetadataRedirect()) { - log("document (" + getMetadataType(metadata) + "): " + metadata.getResolvedName(), level); - Metadata subManifest = fetchManifest(uri, metadata.getArchiveType(), metadata.getArchiveInternalName()); - parseMetadata(uri, subManifest, level); - return; - } - - // internal redirect - if (metadata.isArchiveInternalRedirect()) { - log("document (" + getMetadataType(metadata) + "): " + metadata.getArchiveInternalName(), level); - return; - } - - // single file redirect with external key (only possible if archive manifest or simple redirect but not splitfile) - if (metadata.isSingleFileRedirect()) { - log("document (" + getMetadataType(metadata) + "): " + metadata.getResolvedName(), level); - FreenetURI targetUri = metadata.getSingleTarget(); - log("-> redirect to: " + targetUri, level); - registerManifestUri(targetUri, level); - registerBlockUri(targetUri, true, true, level); - return; - } - - // splitfile - if (metadata.isSplitfile()) { - // splitfile type - if (metadata.isSimpleSplitfile()) { - log("simple splitfile: " + metadata.getResolvedName(), level); - } else { - log("splitfile (not simple): " + metadata.getResolvedName(), level); - } - - // register blocks - Metadata metadata2 = (Metadata) metadata.clone(); - SplitFileSegmentKeys[] segmentKeys = metadata2.grabSegmentKeys(); - for (int i = 0; i < segmentKeys.length; i++) { - int dataBlocks = segmentKeys[i].getDataBlocks(); - int checkBlocks = segmentKeys[i].getCheckBlocks(); - log("segment_" + i + ": " + (dataBlocks + checkBlocks) + - " (data=" + dataBlocks + ", check=" + checkBlocks + ")", level + 1); - for (int j = 0; j < dataBlocks + checkBlocks; j++) { - FreenetURI splitUri = segmentKeys[i].getKey(j, null, false).getURI(); - log("block: " + splitUri, level + 1); - registerBlockUri(splitUri, (j == 0), (j < dataBlocks), level + 1); - } - } - - // create metadata from splitfile (if not simple splitfile) - if (!metadata.isSimpleSplitfile()) { - // TODO: move fetch to net package - FetchContext fetchContext = pr.getHLSimpleClient().getFetchContext(); - ClientContext clientContext = pr.getNode().clientCore.clientContext; - FetchWaiter fetchWaiter = new FetchWaiter(plugin.getFreenetClient()); - List decompressors = new LinkedList<>(); - if (metadata.isCompressed()) { - log("is compressed: " + metadata.getCompressionCodec(), level + 1); - decompressors.add(metadata.getCompressionCodec()); - } else { - log("is not compressed", level + 1); - } - SplitfileGetCompletionCallback cb = new SplitfileGetCompletionCallback(fetchWaiter); - VerySimpleGetter vsg = new VerySimpleGetter((short) 2, null, plugin.getFreenetClient()); - SplitFileFetcher sf = new SplitFileFetcher(metadata, cb, vsg, - fetchContext, true, decompressors, - metadata.getClientMetadata(), 0L, metadata.topDontCompress, - metadata.topCompatibilityMode.code, false, metadata.getResolvedURI(), - true, clientContext); - sf.schedule(clientContext); - - // fetchWaiter.waitForCompletion(); - while (cb.getDecompressedData() == null) { // workaround because in some cases fetchWaiter.waitForCompletion() never finished - if (isInterrupted()) { - return; - } - - if (!isActive()) { - throw new FetchFailedException("Manifest cannot be fetched"); - } - - synchronized (this) { - try { - wait(100); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - } - } - sf.cancel(clientContext); - metadata = fetchManifest(cb.getDecompressedData(), null, null); - parseMetadata(null, metadata, level + 1); - } - } - } - - private String getMetadataType(Metadata metadata) { - try { - - String types = ""; - - if (metadata.isArchiveManifest()) types += ",AM"; - - if (metadata.isSimpleManifest()) types += ",SM"; - - if (metadata.isArchiveInternalRedirect()) types += ",AIR"; - - if (metadata.isArchiveMetadataRedirect()) types += ",AMR"; - - if (metadata.isSymbolicShortlink()) types += ",SSL"; - - if (metadata.isSingleFileRedirect()) types += ",SFR"; - - if (metadata.isSimpleRedirect()) types += ",SR"; - - if (metadata.isMultiLevelMetadata()) types += ",MLM"; - - if (metadata.isSplitfile()) types += ",SF"; - - if (metadata.isSimpleSplitfile()) types += ",SSF"; - - // remove first comma - if (types.length() > 0) types = types.substring(1); - - return types; - - } catch (Exception e) { - plugin.log("Reinserter.getMetadataType(): " + e.getMessage()); - return null; - } - } - - public Plugin getPlugin() { - return plugin; - } - - private static class FetchBlocksResult { - - private int successful = 0; - private int failed = 0; - - void addResult(boolean successful) { - if (successful) { - this.successful++; - } else { - failed++; - } - } - - double calculatePersistenceRate() { - return (double) successful / (successful + failed); - } - } - - private class SplitfileGetCompletionCallback implements GetCompletionCallback { - - private final FetchWaiter fetchWaiter; - private byte[] decompressedSplitFileData = null; - - SplitfileGetCompletionCallback(FetchWaiter fetchWaiter) { - this.fetchWaiter = fetchWaiter; - } - - @Override - public void onFailure(FetchException e, ClientGetState state, ClientContext context) { - fetchWaiter.onFailure(e, null); - } - - @Override - public void onSuccess(StreamGenerator streamGenerator, ClientMetadata clientMetadata, - List decompressors, - ClientGetState state, ClientContext context) { - try { - - // get data - ByteArrayOutputStream rawOutStream = new ByteArrayOutputStream(); - streamGenerator.writeTo(rawOutStream, null); - rawOutStream.close(); - byte[] compressedSplitFileData = rawOutStream.toByteArray(); - - // decompress (if necessary) - if (decompressors.size() > 0) { - try (ByteArrayInputStream compressedInStream = new ByteArrayInputStream(compressedSplitFileData); - ByteArrayOutputStream decompressedOutStream = new ByteArrayOutputStream()) { - decompressors.get(0).decompress(compressedInStream, decompressedOutStream, Integer.MAX_VALUE, -1); - decompressedSplitFileData = decompressedOutStream.toByteArray(); - } - fetchWaiter.onSuccess(null, null); - } else { - decompressedSplitFileData = compressedSplitFileData; - } - - } catch (IOException e) { - plugin.log("SplitfileGetCompletionCallback.onSuccess(): " + e.getMessage()); - } - } - - byte[] getDecompressedData() { - return decompressedSplitFileData; - } - - @Override - public void onBlockSetFinished(ClientGetState state, ClientContext context) { - } - - @Override - public void onExpectedMIME(ClientMetadata metadata, ClientContext context) { - } - - @Override - public void onExpectedSize(long size, ClientContext context) { - } - - @Override - public void onFinalizedMetadata() { - } - - @Override - public void onTransition(ClientGetState oldState, ClientGetState newState, ClientContext context) { - } - - @Override - public void onExpectedTopSize(long size, long compressed, int blocksReq, int blocksTotal, ClientContext context) { - } - - @Override - public void onHashes(HashResult[] hashes, ClientContext context) { - } - - @Override - public void onSplitfileCompatibilityMode(CompatibilityMode min, CompatibilityMode max, byte[] customSplitfileKey, boolean compressed, boolean bottomLayer, boolean definitiveAnyway, ClientContext context) { - } - } - - private static class VerySimpleGetter extends ClientRequester { - - private final FreenetURI uri; - - VerySimpleGetter(short priorityClass, FreenetURI uri, RequestClient rc) { - super(priorityClass, rc); - this.uri = uri; - } - - @Override - public ClientRequestSchedulerGroup getSchedulerGroup() { - return null; - } - - @Override - public FreenetURI getURI() { - return uri; - } - - @Override - public boolean isFinished() { - return false; - } - - @Override - public void onTransition(ClientGetState cgs, ClientGetState cgs1, ClientContext context) { - } - - @Override - public void cancel(ClientContext cc) { - } - - @Override - public void innerNotifyClients(ClientContext cc) { - } - - @Override - protected void innerToNetwork(ClientContext cc) { - } - - @Override - protected ClientBaseCallback getCallback() { - return null; - } - - private static class FakeCallback implements ClientBaseCallback { - - FakeCallback(RequestClient client) { - this.client = client; - } - - final RequestClient client; - - @Override - public void onResume(ClientContext context) { - throw new UnsupportedOperationException(); - } - - @Override - public RequestClient getRequestClient() { - return client; - } - } - } - - private Metadata fetchManifest(FreenetURI uri, ARCHIVE_TYPE archiveType, String manifestName) - throws FetchException, IOException { - FetchResult result = Client.fetch(uri, plugin.getFreenetClient()); - - return fetchManifest(result.asByteArray(), archiveType, manifestName); - } - - private Metadata fetchManifest(byte[] data, ARCHIVE_TYPE archiveType, String manifestName) throws IOException { - Metadata metadata = null; - try (ByteArrayInputStream fetchedDataStream = new ByteArrayInputStream(data)) { - - if (manifestName == null) { - manifestName = ".metadata"; - } - - if (archiveType == null) { - // try to construct metadata directly - try { - metadata = Metadata.construct(data); - } catch (MetadataParseException ignored) { - } - } - - if (metadata == null) { - // unzip and construct metadata - - try { - - InputStream inStream = null; - String entryName = null; - - // get archive stream (try if archive type unknown) - if (archiveType == ARCHIVE_TYPE.TAR || archiveType == null) { - inStream = new TarInputStream(fetchedDataStream); - entryName = ((TarInputStream) inStream).getNextEntry().getName(); - archiveType = ARCHIVE_TYPE.TAR; - } - if (archiveType == ARCHIVE_TYPE.ZIP) { - inStream = new ZipInputStream(fetchedDataStream); - entryName = ((ZipInputStream) inStream).getNextEntry().getName(); - archiveType = ARCHIVE_TYPE.ZIP; - } - - // construct metadata - while (inStream != null && entryName != null) { - if (entryName.equals(manifestName)) { - byte[] buf = new byte[32768]; - ByteArrayOutputStream outStream = new ByteArrayOutputStream(); - int bytes; - while ((bytes = inStream.read(buf)) > 0) { - outStream.write(buf, 0, bytes); - } - outStream.close(); - metadata = Metadata.construct(outStream.toByteArray()); - break; - } - if (archiveType == ARCHIVE_TYPE.TAR) { - entryName = ((TarInputStream) inStream).getNextEntry().getName(); - } else { - entryName = ((ZipInputStream) inStream).getNextEntry().getName(); - } - } - - } catch (Exception e) { - if (archiveType != null) - log("unzip and construct metadata: " + e.getMessage(), 0, 2); - } - } - - if (metadata != null) { - if (archiveType != null) { - manifestName += " (" + archiveType.name() + ")"; - } - metadata.resolve(manifestName); - } - return metadata; - - } - } - - private FreenetURI updateUsk(FreenetURI uri) { - try { - Client.fetch(uri, plugin.getFreenetClient()); - } catch (freenet.client.FetchException e) { - if (e.getMode() == FetchException.FetchExceptionMode.PERMANENT_REDIRECT) { - uri = updateUsk(e.newURI); - } - } - - return uri; - } - - private void registerManifestUri(FreenetURI uri, int level) { - uri = Client.normalizeUri(uri); - if (manifestURIs.containsKey(uri)) { - log("-> already registered manifest", level, 2); - } else { - manifestURIs.put(uri, null); - if (level != -1) { - log("-> registered manifest", level, 2); - } - } - } - - private void registerBlockUri(FreenetURI uri, boolean newSegment, boolean isDataBlock, int logTabLevel) { - if (uri != null) { // uri is null if metadata is created from splitfile - - // no reinsertion for SSK but go to sublevel - if (!uri.isCHK()) { - log("-> no reinsertion of USK, SSK or KSK", logTabLevel, 2); - - // check if uri already reinserted during this session - } else if (blocks.containsKey(Client.normalizeUri(uri))) { - log("-> already registered block", logTabLevel, 2); - - // register - } else { - if (newSegment) { - parsedSegmentId++; - parsedBlockId = -1; - } - uri = Client.normalizeUri(uri); - blocks.put(uri, new Block(uri, parsedSegmentId, ++parsedBlockId, isDataBlock)); - log("-> registered block", logTabLevel, 2); - } - - } - } - - public void registerBlockFetchSuccess(Block block) { - segments.get(block.getSegmentId()).regFetchSuccess(block.isFetchSuccessful()); - } - - public synchronized void updateSegmentStatistic(Segment segment, boolean success) { - String successProp = plugin.getProp("success_segments_" + siteId); - if (success) { - successProp = successProp.substring(0, segment.getId()) + "1" + successProp.substring(segment.getId() + 1); - } - plugin.setProp("success_segments_" + siteId, successProp); - plugin.saveProp(); - } - - public synchronized void updateBlockStatistic(int id, int success, int failed) { - String[] successProp = plugin.getProp("success_" + siteId).split(","); - successProp[id * 2] = String.valueOf(success); - successProp[id * 2 + 1] = String.valueOf(failed); - saveSuccessToProp(successProp); - } - - private void saveSuccessToProp(String[] success) { - StringBuilder newSuccess = new StringBuilder(); - for (int i = 0; i < success.length; i++) { - if (i > 0) { - newSuccess.append(","); - } - newSuccess.append(success[i]); - } - plugin.setProp("success_" + siteId, newSuccess.toString()); - plugin.saveProp(); - } - - public boolean isActive() { - return isActive(false); - } - - private boolean isActive(boolean newActivity) { - if (newActivity) { - lastActivityTime = System.currentTimeMillis(); - return true; - } - if (lastActivityTime != Integer.MIN_VALUE) { - long delay = (System.currentTimeMillis() - lastActivityTime) / 60_000; // delay in minutes - return (delay < SingleJob.MAX_LIFETIME + 5); - } - return false; - } - - public void log(int segmentId, String message, int level, int logLevel) { - StringBuilder buf = new StringBuilder(); - - for (int i = 0; i < level; i++) { - buf.append(" "); - } - - if (segmentId != -1) { - buf.insert(0, "(" + segmentId + ") "); - } - - try { - if (plugin.getIntProp("log_links") == 1) { - int keyPos = message.indexOf("K@"); - if (keyPos != -1) { - keyPos = keyPos - 2; - int keyPos2 = Math.max(message.indexOf(" ", keyPos), message.indexOf("<", keyPos)); - if (keyPos2 == -1) { - keyPos2 = message.length(); - } - String key = message.substring(keyPos, keyPos2); - message = message.substring(0, keyPos) + - "" + key + "" + message.substring(keyPos2); - } - } - } catch (Exception ignored) { - } - - plugin.log(plugin.getLogFilename(siteId), buf.append(message).toString(), logLevel); - } - - public void log(Segment segment, String message, int level, int logLevel) { - log(segment.getId(), message, level, logLevel); - } - - public void log(Segment segment, String message, int level) { - log(segment, message, level, 1); - } - - public void log(String message, int level, int logLevel) { - log(-1, message, level, logLevel); - } - - public void log(String message, int level) { - log(-1, message, level, 1); - } - - public void log(Segment segment, String cMessage, Object obj) { - if (obj != null) { - log(segment, cMessage + " = ok", 1, 2); - } else { - log(segment, cMessage + " = null", 1, 2); - } - } - - public ArrayList getSegments() { - return segments; - } + + private final Plugin plugin; + private final IUriValue uriValue; + private final CountDownLatch latch; + private final PluginRespirator pr; + private final String logFilename; + private final Set manifestURIs = new LinkedHashSet<>(); + private final List segments = new ArrayList<>(); + private long lastActivityTime; + private int parsedSegmentId; + private int parsedBlockId; + + public Reinserter(Plugin plugin, IUriValue uriValue, CountDownLatch latch) { + this.plugin = plugin; + this.uriValue = uriValue; + this.latch = latch; + this.setName(Plugin.PLUGIN_NAME + " ReInserter " + uriValue.getUriId()); + this.pr = plugin.pluginContext.pluginRespirator; + this.logFilename = plugin.getLogFilename(uriValue); + } + + @Override + public void run() { + final int siteId = uriValue.getUriId(); + FreenetURI uri = uriValue.getUri(); + + try { + // init + plugin.log(String.format("start reinserter for site %s (%s)", uri, siteId), 1); + plugin.clearLog(logFilename); + isActive(true); + long startedAt = System.currentTimeMillis(); + long timeLeft = TimeUnit.HOURS.toMillis(plugin.getIntProp(PropertiesKey.SINGLE_URL_TIMESLOT)); + + // update if USK + if (uri.isUSK()) + uri = getNewUsk(uri); + + // check top block availability + checkTopBlockAndRepair(uri); + + // register uri + registerManifestUri(uri, -1); + + // load list of keys (if exists) + // skip if 1 because the manifest failed to fetch before. + final int numBlocks = uriValue.getBlockCount(); + if (numBlocks > 1) { + log("*** loading list of blocks ***", 0, 0); + } else { + // parse metadata + log("*** parsing data structure ***", 0, 0); + parsedSegmentId = -1; + parsedBlockId = -1; + + while (!manifestURIs.isEmpty()) { + if (isInterrupted()) + return; + + if (!isActive()) { + plugin.log("Stop after stuck state (metadata)", 0); + return; + } + + final FreenetURI manifestUri = manifestURIs.iterator().next(); + log(manifestUri.toString(), 0); + + try { + parseMetadata(manifestUri, null, 0); + } catch (final FetchFailedException e) { + log(e.getMessage(), 0); + return; + } + + manifestURIs.remove(manifestUri); + } + + if (isInterrupted()) { + return; + } + + plugin.uriPropsDAO.update(uriValue); + } + + // max segment id + final int maxSegmentId = uriValue.getBlocks().values().stream().mapToInt(IBlock::getSegmentId).max().orElse(-1); + + // init reinsertion + if (uriValue.getSegment() == maxSegmentId) + uriValue.setSegment(-1); + + if (uriValue.getSegment() == -1) { + log("*** starting reinsertion ***", 0, 0); + + // reset success counter + final StringBuilder success = new StringBuilder(); + final StringBuilder segmentsSuccess = new StringBuilder(); + for (int i = 0; i <= maxSegmentId; i++) { + if (i > 0) + success.append(","); + + success.append("0,0"); + segmentsSuccess.append("0"); + } + uriValue.setSuccess(success.toString()); + uriValue.setSuccessSegments(segmentsSuccess.toString()); + + plugin.uriPropsDAO.update(uriValue); + } else { + log("*** continuing reinsertion ***", 0, 0); + + // add dummy segments + for (int i = 0; i <= uriValue.getSegment(); i++) { + segments.add(null); + } + + // reset success counter + final String[] successProp = uriValue.getSuccess().split(","); + for (int i = (uriValue.getSegment() + 1) * 2; i < successProp.length; i++) { + successProp[i] = "0"; + } + + saveSuccessToProp(successProp); + } + + // start reinsertion + boolean doReinsertions = true; + timeLeft -= System.currentTimeMillis() - startedAt; + for (long timeSpent = 0; timeLeft - timeSpent > 0; timeSpent = System.currentTimeMillis() - startedAt, timeLeft -= timeSpent) { + startedAt = System.currentTimeMillis(); + + if (isInterrupted()) { + return; + } + + // next segment + int segmentSize = 0; + for (final IBlock block : uriValue.getBlocks().values()) { + if (block.getSegmentId() == segments.size()) { + segmentSize++; + } + } + if (segmentSize == 0) { + break; // ready + } + final Segment segment = new Segment(this, segments.size(), segmentSize); + for (final IBlock block : uriValue.getBlocks().values()) { + // TODO/FIXME: why is that so that the id is bigger than the size?!? + if ((block.getSegmentId() == segments.size()) && !segment.addBlock(block)) + log(String.format("The BlockId: %s is bigger as the segmentSize: %s -> Block skipped!", block.getId(), segmentSize), 2); + } + + segments.add(segment); + log(segment, "*** segment size: " + segment.size(), 0); + doReinsertions = true; + + // get persistence rate of splitfile segments + if (segment.size() > 1) { + log(segment, "starting availability check for segment (n=" + + plugin.getIntProp(PropertiesKey.SPLITFILE_TEST_SIZE) + ")", 0); + + // select prove blocks + final List requestedBlocks = new ArrayList<>(); + // always fetch exactly the configured number of blocks (or half segment size, whichever is smaller) + final int splitfileTestSize = Math.min( + plugin.getIntProp(PropertiesKey.SPLITFILE_TEST_SIZE), + (int) Math.ceil(segmentSize / 2.0)); + + for (int i = 0; requestedBlocks.size() < splitfileTestSize; i++) { + if (i == segmentSize) { + i = 0; + } + if ((Math.random() < (splitfileTestSize / (double) segmentSize)) + && !(requestedBlocks.contains(segment.getBlock(i)))) { + // add a block + requestedBlocks.add(segment.getBlock(i)); + } + } + + FetchBlocksResult fetchBlocksResult = fetchBlocks(requestedBlocks, segment); + + double persistenceRate = fetchBlocksResult.calculatePersistenceRate(); + if (persistenceRate >= (double) plugin.getIntProp(PropertiesKey.SPLITFILE_TOLERANCE) / 100) { + doReinsertions = false; + segment.regFetchSuccess(persistenceRate); + updateSegmentStatistic(segment, true); + log(segment, "availability of segment ok: " + ((int) (persistenceRate * 100)) + + "% (approximated)", 0, 1); + checkFinishedSegments(); + if (uriValue.getSegment() != maxSegmentId) { + log(segment, "-> segment not reinserted; moving on will resume on next pass.", 0, 1); + break; + } + } else { + log(segment, "availability of segment not ok: " + + ((int) (persistenceRate * 100)) + "% (approximated)", 0, 1); + log(segment, "-> fetch all available blocks now", 0, 1); + } + + // get all available blocks and heal the segment + if (doReinsertions) { + // add the rest of the blocks + for (int i = 0; i < segment.size(); i++) { + if (!(requestedBlocks.contains(segment.getBlock(i)))) { + // add a block + requestedBlocks.add(segment.getBlock(i)); + } + } + + fetchBlocksResult = fetchBlocks(requestedBlocks, segment); + + persistenceRate = fetchBlocksResult.calculatePersistenceRate(); + if (persistenceRate >= plugin.getIntProp(PropertiesKey.SPLITFILE_TOLERANCE) / 100.0) { + doReinsertions = false; + segment.regFetchSuccess(persistenceRate); + updateSegmentStatistic(segment, true); + log(segment, "availability of segment ok: " + ((int) (persistenceRate * 100)) + + "% (exact)", 0, 1); + checkFinishedSegments(); + if (uriValue.getSegment() != maxSegmentId) { + log(segment, "-> segment not reinserted; moving on will resume on next pass.", 0, 1); + break; + } + } else { + log(segment, "availability of segment not ok: " + + ((int) (persistenceRate * 100)) + "% (exact)", 0, 1); + } + } + + if (doReinsertions) { // persistenceRate < splitfile tolerance + // heal segment + + // init + log(segment, "starting segment healing", 0, 1); + final byte[][] dataBlocks = new byte[segment.dataSize()][]; + final byte[][] checkBlocks = new byte[segment.checkSize()][]; + final boolean[] dataBlocksPresent = new boolean[dataBlocks.length]; + final boolean[] checkBlocksPresent = new boolean[checkBlocks.length]; + for (int i = 0; i < dataBlocks.length; i++) { + if (segment.getDataBlock(i).isFetchSuccessful()) { + dataBlocks[i] = segment.getDataBlock(i).getBucket().toByteArray(); + dataBlocksPresent[i] = true; + } else { + dataBlocks[i] = new byte[CHKBlock.DATA_LENGTH]; + dataBlocksPresent[i] = false; + } + } + for (int i = 0; i < checkBlocks.length; i++) { + if (segment.getCheckBlock(i).isFetchSuccessful()) { + checkBlocks[i] = segment.getCheckBlock(i).getBucket().toByteArray(); + checkBlocksPresent[i] = true; + } else { + checkBlocks[i] = new byte[CHKBlock.DATA_LENGTH]; + checkBlocksPresent[i] = false; + } + } + + // decode + final FECCodec codec = FECCodec.getInstance(SplitfileAlgorithm.ONION_STANDARD); + log(segment, "start decoding", 0, 1); + try { + codec.decode(dataBlocks, checkBlocks, dataBlocksPresent, checkBlocksPresent, CHKBlock.DATA_LENGTH); + log(segment, "-> decoding successful", 1, 2); + } catch (final Exception e) { + log(segment, "segment decoding (FEC) failed, do not reinsert", 1, 2); + updateSegmentStatistic(segment, false); + segment.setHealingNotPossible(true); + checkFinishedSegments(); + continue; + } + + // encode (= build all data blocks and check blocks from data blocks) + log(segment, "start encoding", 0, 1); + try { + codec.encode(dataBlocks, checkBlocks, checkBlocksPresent, CHKBlock.DATA_LENGTH); + log(segment, "-> encoding successful", 1, 2); + } catch (final Exception e) { + log(segment, "segment encoding (FEC) failed, do not reinsert", 1, 2); + updateSegmentStatistic(segment, false); + segment.setHealingNotPossible(true); + checkFinishedSegments(); + continue; + } + + // finish + for (int i = 0; i < dataBlocks.length; i++) { + log(segment, "dataBlock_" + i, dataBlocks[i]); + segment.getDataBlock(i).setBucket(new ArrayBucket(dataBlocks[i])); + } + for (int i = 0; i < checkBlocks.length; i++) { + log(segment, "checkBlock_" + i, checkBlocks[i]); + segment.getCheckBlock(i).setBucket(new ArrayBucket(checkBlocks[i])); + } + log(segment, "segment healing (FEC) successful, start with reinsertion", 0, 1); + updateSegmentStatistic(segment, true); + } + } + + // start reinsertion + if (doReinsertions) { + log(segment, "starting reinsertion", 0, 1); + segment.initInsert(); + + final ExecutorService executor = Executors.newFixedThreadPool(plugin.getIntProp(PropertiesKey.POWER)); + try { + for (int i = 0; i < segment.size(); i++) { + checkFinishedSegments(); + isActive(true); + if (segment.size() > 1) { + if (segment.getBlock(i).isFetchSuccessful()) { + segment.regFetchSuccess(true); + } else { + segment.regFetchSuccess(false); + final SingleInsert singleInsert = new SingleInsert(this, segment.getBlock(i)); + executor.execute(singleInsert); + } + } else { + final SingleInsert singleInsert = new SingleInsert(this, segment.getBlock(i)); + executor.execute(singleInsert); + } + } + executor.shutdown(); + final boolean done = executor.awaitTermination(1, TimeUnit.HOURS); + if (!done) { + log(segment, "reinsertion failed", 0); + return; + } + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } finally { + if (!executor.isShutdown()) { + executor.shutdownNow(); + } + } + + } + + // check if segments are finished + checkFinishedSegments(); + } + + // wait for finishing top block, if it was fetched. + if (segments.size() > 0 && segments.get(0) != null) { + while (!(segments.get(0).isFinished())) { + synchronized (this) { + try { + this.wait(1000); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } + + if (isInterrupted()) { + return; + } + + if (!isActive()) { + plugin.log("Stop after stuck state (wait for finishing top block)", 0); + return; + } + + checkFinishedSegments(); + } + } + + // wait for finishing all segments + if (doReinsertions) { + while (uriValue.getSegment() != maxSegmentId) { + synchronized (this) { + try { + this.wait(1_000); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } + + if (isInterrupted()) { + return; + } + + if (!isActive()) { // TODO: this is a bypass + plugin.log("Stop after stuck state (after healing, prop segment_" + siteId + "=" + + uriValue.getSegment() + ", maxSegmentId=" + maxSegmentId + ")", 0); + // TODO: probably segment_siteId prop should be incremented (switched to next segment) + return; + } + + checkFinishedSegments(); + } + } + + // add to history if we've processed the last segment in the file. + if (uriValue.getBlockCount() > 0 && uriValue.getSegment() == maxSegmentId) { + int nPersistence = (int) ((double) plugin.getSuccessValues(uriValue).getSuccess() + / uriValue.getBlockCount() * 100); + String cHistory = uriValue.getHistory(); + String[] aHistory; + if (cHistory == null || cHistory.trim().isEmpty()) { + aHistory = new String[] {}; + } else { + aHistory = cHistory.split(","); + } + final String cThisMonth = (new SimpleDateFormat("MM.yyyy")).format(new Date()); + boolean bNewMonth = true; + if (cHistory != null && cHistory.contains(cThisMonth)) { + bNewMonth = false; + final int nOldPersistence = Integer.parseInt(aHistory[aHistory.length - 1].split("-")[1]); + nPersistence = Math.min(nPersistence, nOldPersistence); + aHistory[aHistory.length - 1] = cThisMonth + "-" + nPersistence; + } + final StringBuilder buf = new StringBuilder(); + for (final String aHistory1 : aHistory) { + if (buf.length() > 0) { + buf.append(","); + } + buf.append(aHistory1); + } + if (bNewMonth) { + if (cHistory != null && cHistory.length() > 0) { + buf.append(","); + } + buf.append(cThisMonth).append("-").append(nPersistence); + } + cHistory = buf.toString(); + + uriValue.setHistory(cHistory); + plugin.uriPropsDAO.update(uriValue); + } + + log("*** reinsertion finished ***", 0, 0); + plugin.log("reinsertion finished for " + uriValue.getUri().toString(), 1); + } catch (final DuplicateKeyException e) { + // if getNewUsk find a new url and this is a duplicate + return; + } catch (final Exception e) { + plugin.log("Reinserter.run()", e); + } finally { + latch.countDown(); + log("stopped", 0); + plugin.log("reinserter stopped (" + siteId + ")"); + } + } + + /** + * Checks for a new USK key edition and returns the key + */ + private FreenetURI getNewUsk(FreenetURI uri) throws DAOException, DuplicateKeyException { + if (uri == null) + return null; + + final FreenetURI newUri = updateUsk(uri); + if (newUri != null && !newUri.equals(uri)) { + plugin.log(String.format("received new uri: %s", newUri), 1); + + // new usk already exists, delete the old + if (plugin.uriPropsDAO.exist(newUri)) { + plugin.log(String.format("remove uri as duplicate: %s", newUri), 1); + plugin.removeUriAndFiles(uriValue); + throw new DuplicateKeyException(); + } + + uriValue.setUri(newUri); + uriValue.setBlockCount(-1); + plugin.uriPropsDAO.update(uriValue); + + return newUri; + } + + return uri; + } + + /** + * Every day this methode checks if the top block is fetchable. + * If it cant be fetched, it tries to insert it again. + */ + private void checkTopBlockAndRepair(FreenetURI uri) throws DAOException { + // get url without any meta informations or ssk key with minus edition + final FreenetURI topBlockUri = Client.normalizeUri(uri.clone()); + + // check only every day + if (plugin.databaseDAO.lastAccessDiff(topBlockUri) <= TimeUnit.DAYS.toMillis(1)) + return; + + log("checkTopBlockAndRepair", 2); + + try { + Client.fetch(topBlockUri, plugin.getFreenetClient()); + } catch (final FetchException e) { + log(e.getShortMessage(), 0, 0); + + try { + // there was a problem fetching the top block, so we try to insert it again + final IDatabaseBlock databaseBlock = plugin.databaseDAO.read(topBlockUri); + FreenetURI insertUri = null; + if (databaseBlock != null) { + insertUri = Client.insert(topBlockUri, databaseBlock.getData(), plugin.getFreenetClient()); + + if (insertUri != null) { + if (topBlockUri.equals(insertUri)) { + log("Successfully inserted top block: " + insertUri.toString(), 0); + } else { + log("Top block insertion failed - different uri: " + insertUri.toString(), 0); + } + } else { + log("Top block insertion failed (insertUri = null)", 0); + } + } else { + log("Top block insertion failed (no saved block)", 0); + } + } catch (final InsertException e1) { + log(e1.getMessage(), 0, 0); + } + } + + plugin.databaseDAO.lastAccessUpdate(topBlockUri); + } + + private FetchBlocksResult fetchBlocks(List requestedBlocks, Segment segment) { + final ExecutorService executor = Executors.newFixedThreadPool(plugin.getIntProp(PropertiesKey.POWER)); + + final FetchBlocksResult fetchBlocksResult = new FetchBlocksResult(); + try { + final List tasks = requestedBlocks.stream().map(requestedBlock -> new SingleFetch(this, requestedBlock)).collect(Collectors.toList()); + final List> results = executor.invokeAll(tasks, 1, TimeUnit.HOURS); + for (final Future future : results) + fetchBlocksResult.addResult(future.get()); + + executor.shutdown(); + final boolean done = executor.awaitTermination(1, TimeUnit.HOURS); + if (!done) { + log(segment, "availability check failed", 0); + return fetchBlocksResult; + } + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (final ExecutionException e) { + plugin.log("Error fetchBlocks", e); + } finally { + if (!executor.isShutdown()) + executor.shutdownNow(); + } + + return fetchBlocksResult; + } + + private void checkFinishedSegments() throws DAOException { + int segment; + while ((segment = uriValue.getSegment()) < segments.size() - 1) { + if (!segments.get(segment + 1).isFinished()) { + break; + } + uriValue.setSegment(segment + 1); + } + + plugin.uriPropsDAO.update(uriValue); + } + + private void parseMetadata(FreenetURI uri, Metadata metadata, int level) throws FetchFailedException, MetadataParseException, FetchException, IOException, DAOException { + if (isInterrupted()) + return; + + // register uri + registerBlockUri(uri, true, true, level); + + // constructs top level simple manifest (= first action on a new uri) + if (metadata == null) { + final FetchResult fetchResult = Client.fetch(uri, plugin.getFreenetClient()); + final byte[] fetchBytes = fetchResult.asByteArray(); + + if (fetchBytes.length > IDatabaseBlock.MAX_SIZE) { + log(String.format("parseMetadata: block is too big (%s) skipped -> Mime: %s", fetchBytes.length, fetchResult.getMimeType()), level); + } else { + // create or update block + final IDatabaseBlock databaseBlock = plugin.databaseDAO.read(uri); + if (databaseBlock == null) { + plugin.databaseDAO.create(uri, fetchBytes); + } else { + databaseBlock.setData(fetchBytes); + plugin.databaseDAO.update(databaseBlock); + } + } + + metadata = fetchManifest(fetchBytes, null, null); + if (metadata == null) { + log("no metadata", level); + return; + } + + log(String.format("parseMetadata: metadata: %s", metadata.dump()), level); + } + + // internal manifest (simple manifest) + if (metadata.isSimpleManifest()) { + log("manifest (" + getMetadataType(metadata) + "): " + metadata.getResolvedName(), level); + HashMap targetList = null; + try { + targetList = metadata.getDocuments(); + } catch (final Exception ignored) {} + + if (targetList != null) { + for (final Entry entry : targetList.entrySet()) { + if (isInterrupted()) { + return; + } + // get document + final Metadata target = entry.getValue(); + // remember document name + target.resolve(entry.getKey()); + // parse document + parseMetadata(uri, target, level + 1); + } + } + + return; + } + + // redirect to submanifest + if (metadata.isArchiveMetadataRedirect()) { + log("document (" + getMetadataType(metadata) + "): " + metadata.getResolvedName(), level); + final Metadata subManifest = fetchManifest(uri, metadata.getArchiveType(), metadata.getArchiveInternalName()); + parseMetadata(uri, subManifest, level); + return; + } + + // internal redirect + if (metadata.isArchiveInternalRedirect()) { + log("document (" + getMetadataType(metadata) + "): " + metadata.getArchiveInternalName(), level); + return; + } + + // single file redirect with external key (only possible if archive manifest or simple redirect but not splitfile) + if (metadata.isSingleFileRedirect()) { + log("document (" + getMetadataType(metadata) + "): " + metadata.getResolvedName(), level); + final FreenetURI targetUri = metadata.getSingleTarget(); + log("-> redirect to: " + targetUri, level); + registerManifestUri(targetUri, level); + registerBlockUri(targetUri, true, true, level); + return; + } + + // splitfile + if (metadata.isSplitfile()) { + // splitfile type + if (metadata.isSimpleSplitfile()) { + log("simple splitfile: " + metadata.getResolvedName(), level); + } else { + log("splitfile (not simple): " + metadata.getResolvedName(), level); + } + + // register blocks + final Metadata metadata2 = (Metadata) metadata.clone(); + final SplitFileSegmentKeys[] segmentKeys = metadata2.grabSegmentKeys(); + for (int i = 0; i < segmentKeys.length; i++) { + final int dataBlocks = segmentKeys[i].getDataBlocks(); + final int checkBlocks = segmentKeys[i].getCheckBlocks(); + log("segment_" + i + ": " + (dataBlocks + checkBlocks) + + " (data=" + dataBlocks + ", check=" + checkBlocks + ")", level + 1); + for (int j = 0; j < dataBlocks + checkBlocks; j++) { + final FreenetURI splitUri = segmentKeys[i].getKey(j, null, false).getURI(); + log("block: " + splitUri, level + 1); + registerBlockUri(splitUri, (j == 0), (j < dataBlocks), level + 1); + } + } + + // create metadata from splitfile (if not simple splitfile) + if (!metadata.isSimpleSplitfile()) { + // TODO: move fetch to net package + final FetchContext fetchContext = pr.getHLSimpleClient().getFetchContext(); + final ClientContext clientContext = pr.getNode().clientCore.clientContext; + final FetchWaiter fetchWaiter = new FetchWaiter(plugin.getFreenetClient()); + final List decompressors = new LinkedList<>(); + if (metadata.isCompressed()) { + log("is compressed: " + metadata.getCompressionCodec(), level + 1); + decompressors.add(metadata.getCompressionCodec()); + } else { + log("is not compressed", level + 1); + } + final SplitfileGetCompletionCallback cb = new SplitfileGetCompletionCallback(fetchWaiter, plugin); + final VerySimpleGetter vsg = new VerySimpleGetter((short) 2, null, plugin.getFreenetClient()); + final SplitFileFetcher sf = new SplitFileFetcher(metadata, cb, vsg, + fetchContext, true, decompressors, + metadata.getClientMetadata(), 0L, metadata.topDontCompress, + metadata.topCompatibilityMode.code, false, metadata.getResolvedURI(), + true, clientContext); + sf.schedule(clientContext); + + // fetchWaiter.waitForCompletion(); + while (cb.getDecompressedData() == null) { // workaround because in some cases fetchWaiter.waitForCompletion() never finished + if (isInterrupted()) { + return; + } + + if (!isActive()) { + throw new FetchFailedException("Manifest cannot be fetched"); + } + + synchronized (this) { + try { + wait(100); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } + } + sf.cancel(clientContext); + metadata = fetchManifest(cb.getDecompressedData(), null, null); + parseMetadata(null, metadata, level + 1); + } + } + } + + private String getMetadataType(Metadata metadata) { + try { + + String types = ""; + + if (metadata.isArchiveManifest()) + types += ",AM"; + + if (metadata.isSimpleManifest()) + types += ",SM"; + + if (metadata.isArchiveInternalRedirect()) + types += ",AIR"; + + if (metadata.isArchiveMetadataRedirect()) + types += ",AMR"; + + if (metadata.isSymbolicShortlink()) + types += ",SSL"; + + if (metadata.isSingleFileRedirect()) + types += ",SFR"; + + if (metadata.isSimpleRedirect()) + types += ",SR"; + + if (metadata.isMultiLevelMetadata()) + types += ",MLM"; + + if (metadata.isSplitfile()) + types += ",SF"; + + if (metadata.isSimpleSplitfile()) + types += ",SSF"; + + // remove first comma + if (types.length() > 0) + types = types.substring(1); + + return types; + } catch (final Exception e) { + plugin.log("Reinserter.getMetadataType(): " + e.getMessage(), e); + return null; + } + } + + public Plugin getPlugin() { + return plugin; + } + + private Metadata fetchManifest(FreenetURI uri, ARCHIVE_TYPE archiveType, String manifestName) + throws FetchException, IOException { + final FetchResult result = Client.fetch(uri, plugin.getFreenetClient()); + + return fetchManifest(result.asByteArray(), archiveType, manifestName); + } + + private Metadata fetchManifest(byte[] data, ARCHIVE_TYPE archiveType, String manifestName) throws IOException { + Metadata metadata = null; + + try (ByteArrayInputStream fetchedDataStream = new ByteArrayInputStream(data)) { + if (manifestName == null) + manifestName = ".metadata"; + + if (archiveType == null) { + // try to construct metadata directly + try { + metadata = Metadata.construct(data); + } catch (final MetadataParseException ex) {/* ignored */} + } + + if (metadata == null) { + // unzip and construct metadata + try { + InputStream inStream = null; + String entryName = null; + + // get archive stream (try if archive type unknown) + if (archiveType == ARCHIVE_TYPE.TAR || archiveType == null) { + inStream = new TarInputStream(fetchedDataStream); + entryName = ((TarInputStream) inStream).getNextEntry().getName(); + archiveType = ARCHIVE_TYPE.TAR; + } + if (archiveType == ARCHIVE_TYPE.ZIP) { + inStream = new ZipInputStream(fetchedDataStream); + entryName = ((ZipInputStream) inStream).getNextEntry().getName(); + archiveType = ARCHIVE_TYPE.ZIP; + } + + // construct metadata + while (inStream != null && entryName != null) { + if (entryName.equals(manifestName)) { + final byte[] buf = new byte[32768]; + final ByteArrayOutputStream outStream = new ByteArrayOutputStream(); + int bytes; + while ((bytes = inStream.read(buf)) > 0) { + outStream.write(buf, 0, bytes); + } + outStream.close(); + metadata = Metadata.construct(outStream.toByteArray()); + break; + } + if (archiveType == ARCHIVE_TYPE.TAR) { + entryName = ((TarInputStream) inStream).getNextEntry().getName(); + } else { + entryName = ((ZipInputStream) inStream).getNextEntry().getName(); + } + } + } catch (final Exception e) { + if (archiveType != null) + log("unzip and construct metadata: " + e.getMessage(), 0, 2); + } + } + + if (metadata != null) { + if (archiveType != null) { + manifestName += " (" + archiveType.name() + ")"; + } + + metadata.resolve(manifestName); + } + + return metadata; + } + } + + private FreenetURI updateUsk(FreenetURI uri) { + try { + Client.fetch(uri, plugin.getFreenetClient()); + } catch (final freenet.client.FetchException e) { + if (e.getMode() == FetchException.FetchExceptionMode.PERMANENT_REDIRECT) { + uri = updateUsk(e.newURI); + } + } + + return uri; + } + + /** + * Checks if a uri is already registered as manifest + */ + private void registerManifestUri(FreenetURI uri, int level) { + // get url without any meta informations or ssk key with minus edition + final FreenetURI normalizeUri = Client.normalizeUri(uri.clone()); + if (manifestURIs.contains(normalizeUri)) { + log("-> already registered manifest", level, 2); + } else { + manifestURIs.add(normalizeUri); + if (level != -1) + log("-> registered manifest", level, 2); + } + } + + private void registerBlockUri(FreenetURI uri, boolean newSegment, boolean isDataBlock, int logTabLevel) { + // uri is null if metadata is created from splitfile + if (uri == null) + return; + + // no reinsertion for SSK but go to sublevel + if (!uri.isCHK()) { + log("-> no reinsertion of USK, SSK or KSK", logTabLevel, 2); + } else if (uriValue.getBlocks().containsKey(Client.normalizeUri(uri))) { + // check if uri already reinserted during this session + log("-> already registered block", logTabLevel, 2); + } else { + // register + if (newSegment) { + parsedSegmentId++; + parsedBlockId = -1; + } + uri = Client.normalizeUri(uri); + uriValue.getBlocks().put(uri, new Block(uri, parsedSegmentId, ++parsedBlockId, isDataBlock)); + log("-> registered block", logTabLevel, 2); + } + } + + public void registerBlockFetchSuccess(IBlock block) { + try { + segments.get(block.getSegmentId()).regFetchSuccess(block.isFetchSuccessful()); + } catch (final DAOException e) { + plugin.log("Problem with a DAO at Block: %s", e, block); + } + } + + public synchronized void updateSegmentStatistic(Segment segment, boolean success) throws DAOException { + String successProp = uriValue.getSuccessSegments(); + if (success) { + successProp = successProp.substring(0, segment.getId()) + "1" + successProp.substring(segment.getId() + 1); + } + uriValue.setSuccessSegments(successProp); + plugin.uriPropsDAO.update(uriValue); + } + + public synchronized void updateBlockStatistic(int id, int success, int failed) throws DAOException { + final String[] successProp = uriValue.getSuccess().split(","); + successProp[id * 2] = String.valueOf(success); + successProp[id * 2 + 1] = String.valueOf(failed); + saveSuccessToProp(successProp); + } + + private void saveSuccessToProp(String[] success) throws DAOException { + final String newSuccess = String.join(",", success); + uriValue.setSuccess(newSuccess); + plugin.uriPropsDAO.update(uriValue); + } + + public boolean isActive() { + return isActive(false); + } + + private boolean isActive(boolean newActivity) { + if (newActivity) { + lastActivityTime = System.currentTimeMillis(); + return true; + } + if (lastActivityTime != Integer.MIN_VALUE) { + final long delay = (System.currentTimeMillis() - lastActivityTime) / 60_000; // delay in minutes + return (delay < SingleJob.MAX_LIFETIME + 5); + } + return false; + } + + public void log(int segmentId, String message, int level, int logLevel) { + final StringBuilder buf = new StringBuilder(); + + for (int i = 0; i < level; i++) { + buf.append(" "); + } + + if (segmentId != -1) { + buf.insert(0, "(" + segmentId + ") "); + } + + try { + if (plugin.getIntProp(PropertiesKey.LOG_LINKS) == 1) { + int keyPos = message.indexOf("K@"); + if (keyPos != -1) { + keyPos = keyPos - 2; + int keyPos2 = Math.max(message.indexOf(" ", keyPos), message.indexOf("<", keyPos)); + if (keyPos2 == -1) { + keyPos2 = message.length(); + } + final String key = message.substring(keyPos, keyPos2); + message = message.substring(0, keyPos) + + "" + key + "" + message.substring(keyPos2); + } + } + } catch (final Exception ex) {/* ignore */} + + plugin.logFile(logFilename, buf.append(message).toString(), logLevel); + } + + public void log(Segment segment, String message, int level, int logLevel) { + log(segment.getId(), message, level, logLevel); + } + + public void log(Segment segment, String message, int level) { + log(segment, message, level, 1); + } + + public void log(String message, int level, int logLevel) { + log(-1, message, level, logLevel); + } + + public void log(String message, int level) { + log(-1, message, level, 1); + } + + public void log(Segment segment, String cMessage, Object obj) { + if (obj != null) { + log(segment, cMessage + " = ok", 1, 2); + } else { + log(segment, cMessage + " = null", 1, 2); + } + } + + public List getSegments() { + return segments; + } + } diff --git a/src/keepalive/service/reinserter/SplitfileGetCompletionCallback.java b/src/keepalive/service/reinserter/SplitfileGetCompletionCallback.java new file mode 100644 index 0000000..45893a5 --- /dev/null +++ b/src/keepalive/service/reinserter/SplitfileGetCompletionCallback.java @@ -0,0 +1,94 @@ +package keepalive.service.reinserter; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.List; + +import freenet.client.ClientMetadata; +import freenet.client.FetchException; +import freenet.client.FetchWaiter; +import freenet.client.InsertContext.CompatibilityMode; +import freenet.client.async.ClientContext; +import freenet.client.async.ClientGetState; +import freenet.client.async.GetCompletionCallback; +import freenet.client.async.StreamGenerator; +import freenet.crypt.HashResult; +import freenet.support.compress.Compressor; +import keepalive.Plugin; + +class SplitfileGetCompletionCallback implements GetCompletionCallback { + + private final FetchWaiter fetchWaiter; + private byte[] decompressedSplitFileData = null; + private final Plugin plugin; + + SplitfileGetCompletionCallback(FetchWaiter fetchWaiter, Plugin plugin) { + this.fetchWaiter = fetchWaiter; + this.plugin = plugin; + } + + @Override + public void onFailure(FetchException e, ClientGetState state, ClientContext context) { + fetchWaiter.onFailure(e, null); + } + + @Override + public void onSuccess(StreamGenerator streamGenerator, ClientMetadata clientMetadata, + List decompressors, + ClientGetState state, ClientContext context) { + try { + + // get data + final ByteArrayOutputStream rawOutStream = new ByteArrayOutputStream(); + streamGenerator.writeTo(rawOutStream, null); + rawOutStream.close(); + final byte[] compressedSplitFileData = rawOutStream.toByteArray(); + + // decompress (if necessary) + if (decompressors.size() > 0) { + try (ByteArrayInputStream compressedInStream = new ByteArrayInputStream(compressedSplitFileData); + ByteArrayOutputStream decompressedOutStream = new ByteArrayOutputStream()) { + decompressors.get(0).decompress(compressedInStream, decompressedOutStream, Integer.MAX_VALUE, -1); + decompressedSplitFileData = decompressedOutStream.toByteArray(); + } + fetchWaiter.onSuccess(null, null); + } else { + decompressedSplitFileData = compressedSplitFileData; + } + + } catch (final IOException e) { + plugin.log("SplitfileGetCompletionCallback.onSuccess(): " + e.getMessage()); + } + } + + byte[] getDecompressedData() { + return decompressedSplitFileData; + } + + @Override + public void onBlockSetFinished(ClientGetState state, ClientContext context) {} + + @Override + public void onExpectedMIME(ClientMetadata metadata, ClientContext context) {} + + @Override + public void onExpectedSize(long size, ClientContext context) {} + + @Override + public void onFinalizedMetadata() {} + + @Override + public void onTransition(ClientGetState oldState, ClientGetState newState, ClientContext context) {} + + @Override + public void onExpectedTopSize(long size, long compressed, int blocksReq, int blocksTotal, ClientContext context) {} + + @Override + public void onHashes(HashResult[] hashes, ClientContext context) {} + + @Override + public void onSplitfileCompatibilityMode(CompatibilityMode min, CompatibilityMode max, byte[] customSplitfileKey, boolean compressed, boolean bottomLayer, boolean definitiveAnyway, + ClientContext context) {} + +} diff --git a/src/keepalive/service/reinserter/VerySimpleGetter.java b/src/keepalive/service/reinserter/VerySimpleGetter.java new file mode 100644 index 0000000..3cfdcc3 --- /dev/null +++ b/src/keepalive/service/reinserter/VerySimpleGetter.java @@ -0,0 +1,53 @@ +package keepalive.service.reinserter; + +import freenet.client.async.ClientBaseCallback; +import freenet.client.async.ClientContext; +import freenet.client.async.ClientGetState; +import freenet.client.async.ClientRequestSchedulerGroup; +import freenet.client.async.ClientRequester; +import freenet.keys.FreenetURI; +import freenet.node.RequestClient; + +class VerySimpleGetter extends ClientRequester { + + private static final long serialVersionUID = -4860494764560738604L; + private final FreenetURI uri; + + VerySimpleGetter(short priorityClass, FreenetURI uri, RequestClient rc) { + super(priorityClass, rc); + this.uri = uri; + } + + @Override + public ClientRequestSchedulerGroup getSchedulerGroup() { + return null; + } + + @Override + public FreenetURI getURI() { + return uri; + } + + @Override + public boolean isFinished() { + return false; + } + + @Override + public void onTransition(ClientGetState cgs, ClientGetState cgs1, ClientContext context) {} + + @Override + public void cancel(ClientContext cc) {} + + @Override + public void innerNotifyClients(ClientContext cc) {} + + @Override + protected void innerToNetwork(ClientContext cc) {} + + @Override + protected ClientBaseCallback getCallback() { + return null; + } + +} diff --git a/src/keepalive/urivalues/IUriValue.java b/src/keepalive/urivalues/IUriValue.java new file mode 100644 index 0000000..429c0ca --- /dev/null +++ b/src/keepalive/urivalues/IUriValue.java @@ -0,0 +1,49 @@ +package keepalive.urivalues; + +import java.util.Map; + +import freenet.keys.FreenetURI; +import keepalive.model.IBlock; + +public interface IUriValue { + + int getUriId(); + + FreenetURI getUri(); + + void setUri(FreenetURI uri); + + int getBlockCount(); + + void setBlockCount(int blocks); + + Map getBlocks(); + + void setBlocks(Map blocks); + + String getSuccessSegments(); + + void setSuccessSegments(String successSegments); + + String getSuccess(); + + void setSuccess(String success); + + String getHistory(); + + void setHistory(String history); + + int getSegment(); + + void setSegment(int segment); + + default String getShortUri() { + final FreenetURI uri = getUri(); + if (uri == null) + return ""; + + final String strUri = uri.toString(); + return strUri.substring(0, 20) + "...." + strUri.substring(strUri.length() - 50); + } + +} diff --git a/src/keepalive/urivalues/IUriValuesDAO.java b/src/keepalive/urivalues/IUriValuesDAO.java new file mode 100644 index 0000000..5e18a00 --- /dev/null +++ b/src/keepalive/urivalues/IUriValuesDAO.java @@ -0,0 +1,59 @@ +package keepalive.urivalues; + +import java.util.List; + +import freenet.keys.FreenetURI; +import keepalive.exceptions.DAOException; + +/** + * Interface for a DAO that persist all values around {@link IUriValue}
+ * CRUD style methods for interaction with {@link IUriValue} + */ +public interface IUriValuesDAO { + + /** + * Saves a new {@link IUriValue} with a new unique {@link IUriValue#getUriId()} + * @param uri the freenet key + * @return the saved value (with the new unique uriId) + * @throws DAOException problems like already saved + */ + IUriValue create(FreenetURI uri) throws DAOException; + + /** + * Gets a {@link IUriValue} from the save + * @param uriId the unique uriId + * @return the saved value + * @throws DAOException if it doesnt exist + */ + IUriValue read(int uriId) throws DAOException; + + /** + * Updates a {@link IUriValue} + * @param uriValue the uriValue with all of its values + * @throws DAOException if it doesnt exist + */ + void update(IUriValue uriValue) throws DAOException; + + /** + * Deletes a {@link IUriValue} from the save + * @param uriId the unique uriId + * @throws DAOException if it doesnt exist + */ + void delete(int uriId) throws DAOException; + + /** + * Checks if a {@link IUriValue} with this freenetUri is already saved + * @param freenetUri the freenetUri + * @return if the freenetUri is already saved + * @throws DAOException + */ + boolean exist(FreenetURI freenetUri) throws DAOException; + + /** + * Returns a list with all saved {@link IUriValue}s + * @return + * @throws DAOException + */ + List getAll() throws DAOException; + +} diff --git a/src/keepalive/urivalues/impl/PropertiesUriValuesDAO.java b/src/keepalive/urivalues/impl/PropertiesUriValuesDAO.java new file mode 100644 index 0000000..366b0b6 --- /dev/null +++ b/src/keepalive/urivalues/impl/PropertiesUriValuesDAO.java @@ -0,0 +1,293 @@ +package keepalive.urivalues.impl; + +import java.io.File; +import java.net.MalformedURLException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; +import java.util.stream.Collectors; + +import freenet.keys.FreenetURI; +import keepalive.Plugin; +import keepalive.exceptions.DAOException; +import keepalive.model.Block; +import keepalive.model.IBlock; +import keepalive.model.PropertiesKey; +import keepalive.urivalues.IUriValue; +import keepalive.urivalues.IUriValuesDAO; + +/** + * Implementation of {@link IUriValuesDAO}
+ * The uri values get managed and persisted with the plugin properties + */ +public class PropertiesUriValuesDAO implements IUriValuesDAO { + + private final Plugin plugin; + private final Properties prop; + + public PropertiesUriValuesDAO(Plugin plugin, Properties prop) { + this.plugin = plugin; + this.prop = prop; + } + + @Override + public IUriValue create(FreenetURI uri) throws DAOException { + if (uri == null) + throw new DAOException("The uri need to be not null!"); + if (existUri(uri)) + throw new DAOException("The uri: '%s' is already saved", uri); + + final IUriValue uriProps = new UriValue(getNewUriId(), uri); + final int uriId = uriProps.getUriId(); + + setProp(PropertiesKey.URI, uriId, uriProps.getUri().toString()); + setIntProp(PropertiesKey.BLOCKS, uriId, uriProps.getBlockCount()); + setProp(PropertiesKey.SUCCESS, uriId, uriProps.getSuccess()); + setIntProp(PropertiesKey.SEGMENT, uriId, uriProps.getSegment()); + + plugin.setProp(PropertiesKey.IDS, plugin.getProp(PropertiesKey.IDS) + uriProps.getUriId() + ","); + + plugin.saveProp(); + return uriProps; + } + + @Override + public UriValue read(int uriId) throws DAOException { + if (!existUriId(uriId)) + throw new DAOException("The uriId: '%s' doesnt exist", uriId); + + UriValue result = null; + try { + final UriValue uriValue = new UriValue(uriId); + + uriValue.setUri(new FreenetURI(getProp(PropertiesKey.URI, uriId))); + uriValue.setBlockCount(getIntProp(PropertiesKey.BLOCKS, uriId)); + loadBlockUris(uriValue); + uriValue.setSuccessSegments(getProp(PropertiesKey.SUCCESS_SEGMENTS, uriId)); + uriValue.setSuccess(getProp(PropertiesKey.SUCCESS, uriId)); + uriValue.setHistory(getProp(PropertiesKey.HISTORY, uriId)); + uriValue.setSegment(getIntProp(PropertiesKey.SEGMENT, uriId)); + + result = uriValue; + } catch (final MalformedURLException e) { + throw new DAOException("The uriId: '%s' couldnt be loaded, because the uri seams broken", e, uriId); + } + + return result; + } + + @Override + public void update(IUriValue uriValue) throws DAOException { + if (uriValue == null) + throw new DAOException("The uriValue need to be not null!"); + if (!exist(uriValue.getUri())) + throw new DAOException("The uri: '%s' doesnt exist", uriValue.getUri()); + + final int uriId = uriValue.getUriId(); + setProp(PropertiesKey.URI, uriId, uriValue.getUri().toString()); + saveBlockUris(uriValue); + setIntProp(PropertiesKey.BLOCKS, uriId, uriValue.getBlocks().size() > 0 ? uriValue.getBlocks().size() : uriValue.getBlockCount()); + setProp(PropertiesKey.SUCCESS_SEGMENTS, uriId, uriValue.getSuccessSegments()); + setProp(PropertiesKey.SUCCESS, uriId, uriValue.getSuccess()); + setProp(PropertiesKey.HISTORY, uriId, uriValue.getHistory()); + setIntProp(PropertiesKey.SEGMENT, uriId, uriValue.getSegment()); + + plugin.saveProp(); + } + + @Override + public void delete(int uriId) throws DAOException { + if (!existUriId(uriId)) + throw new DAOException("The uriId: '%s' doesnt exist", uriId); + + removeProp(PropertiesKey.URI, uriId); + removeProp(PropertiesKey.BLOCKS, uriId); + removeProp(PropertiesKey.SUCCESS, uriId); + removeProp(PropertiesKey.SUCCESS_SEGMENTS, uriId); + removeProp(PropertiesKey.SEGMENT, uriId); + removeProp(PropertiesKey.HISTORY, uriId); + + // remove key files + final File file = new File(plugin.getPluginDirectory() + getBlockListFilename(uriId)); + if (file.exists() && !file.delete()) { + plugin.log("delete(): remove key files was not successful.", 1); + } + + final String ids = ("," + plugin.getProp(PropertiesKey.IDS)).replaceAll("," + uriId + ",", ","); + plugin.setProp(PropertiesKey.IDS, ids.substring(1)); + + plugin.saveProp(); + } + + @Override + public boolean exist(FreenetURI freenetUri) throws DAOException { + if (freenetUri == null) + throw new DAOException("The freenetUri need to be not null!"); + + return existUri(freenetUri); + } + + @Override + public List getAll() throws DAOException { + return getAllUriIds().parallelStream() + .map(this::readUnchecked) + .filter(x -> x != null && x.getUri() != null) + .collect(Collectors.toList()); + } + + private boolean existUri(FreenetURI uri) throws DAOException { + if (uri == null) + throw new DAOException("The uri need to be not null!"); + + return getAllUriIds().stream().map(x -> getProp(PropertiesKey.URI, x)).anyMatch(x -> uri.toString().equalsIgnoreCase(x)); + } + + private boolean existUriId(int uriId) throws DAOException { + return getAllUriIds().stream().anyMatch(x -> x.equals(uriId)); + } + + private List getAllUriIds() throws DAOException { + List result = new ArrayList<>(); + + try { + final String uriIds = plugin.getProp(PropertiesKey.IDS); + if (uriIds == null || uriIds.trim().isEmpty()) + return result; + + result = Arrays.asList(uriIds.split(",")).stream().map(Integer::parseInt).collect(Collectors.toList()); + } catch (final NumberFormatException e) { + throw new DAOException("There is a problen with the id parsing", e); + } + + return result; + } + + /** + * Returns the highest free id or 0 + * @throws DAOException + */ + private int getNewUriId() throws DAOException { + return getAllUriIds().stream().mapToInt(x -> x).max().orElse(-1) + 1; + } + + /** + * Throws no exception for use in streams + */ + private UriValue readUnchecked(int uriId) { + try { + return read(uriId); + } catch (final Exception e) { + plugin.log("Error on readUnchecked with uriId: %s", e, uriId); + return null; + } + } + + private void saveBlockUris(IUriValue uriValue) throws DAOException { + final Path fileName = Paths.get(plugin.getPluginDirectory(), getBlockListFilename(uriValue.getUriId())); + final List content = uriValue.getBlocks().values().stream().map(this::convertLine).collect(Collectors.toList()); + + try { + final OpenOption openOption = Files.exists(fileName) ? StandardOpenOption.WRITE : StandardOpenOption.CREATE; + Files.write(fileName, content, StandardCharsets.UTF_8, openOption, StandardOpenOption.TRUNCATE_EXISTING); + } catch (final Exception e) { + throw new DAOException("Error saving blocks for: %s", e, uriValue.getUri()); + } + } + + private void loadBlockUris(IUriValue uriValue) throws DAOException { + final Path fileName = Paths.get(plugin.getPluginDirectory(), getBlockListFilename(uriValue.getUriId())); + if (!Files.exists(fileName)) + return; + + try { + Files.readAllLines(fileName, StandardCharsets.UTF_8).stream() + .filter(x -> x != null && !x.trim().isEmpty() && x.contains("#")) + .map(this::parseLine) + .filter(x -> x != null) + .forEach(x -> uriValue.getBlocks().put(x.getUri(), x)); + } catch (final Exception e) { + throw new DAOException("Error loading blocks for: %s", e, uriValue.getUri()); + } + } + + private String convertLine(IBlock block) { + final String type = block.isDataBlock() ? "d" : "c"; + final String msg = String.format("%s#%s#%s#%s", block.getUri(), block.getSegmentId(), block.getId(), type); + if (msg.startsWith("HK")) + plugin.logF("convertLine: %s | %s", block.getUri(), msg); + return msg; + } + + private Block parseLine(String fileLine) { + try { + final String[] values = fileLine.split("#", 4); + return new Block(new FreenetURI(values[0]), Integer.parseInt(values[1]), Integer.parseInt(values[2]), "d".equals(values[3])); + } catch (NumberFormatException | MalformedURLException e) { + plugin.log("parseLine error: %s", e, fileLine); + return null; + } + } + + private String getBlockListFilename(int uriId) { + return String.format("keys%s.txt", uriId); + } + + /** + * special use for properties for a specific id + * {@link Properties#getProperty(String)} + */ + private String getProp(PropertiesKey key, int id) { + return key != null ? prop.getProperty(key.toString() + "_" + id) : null; + } + + /** + * special use for properties for a specific id + * {@link Properties#setProperty(String, String)} + */ + private void setProp(PropertiesKey key, int id, String value) { + if (key == null || value == null) + return; + + prop.setProperty(key.toString() + "_" + id, value); + } + + /** + * special use for properties for a specific id + * {@link Properties#remove(Object)} + */ + private void removeProp(PropertiesKey key, int id) { + if (key == null) + return; + + prop.remove(key.toString() + "_" + id); + } + + /** + * get a value and parse it to an integer
+ * special use for properties for a specific id + */ + private int getIntProp(PropertiesKey key, int id) { + final String strValue = getProp(key, id); + return plugin.parseInt(strValue); + } + + /** + * special use for properties for a specific id + * {@link Properties#getProperty(String)} + */ + private void setIntProp(PropertiesKey key, int id, int value) { + final String strValue = String.valueOf(value); + if (key == null || strValue == null) + return; + + setProp(key, id, strValue); + } + +} diff --git a/src/keepalive/urivalues/impl/UriValue.java b/src/keepalive/urivalues/impl/UriValue.java new file mode 100644 index 0000000..246d780 --- /dev/null +++ b/src/keepalive/urivalues/impl/UriValue.java @@ -0,0 +1,141 @@ +package keepalive.urivalues.impl; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +import freenet.keys.FreenetURI; +import keepalive.model.IBlock; +import keepalive.urivalues.IUriValue; +import keepalive.urivalues.IUriValuesDAO; + +/** + * Implementation of {@link IUriValue}
+ * This is the default implementation for an entity that is managed be the {@link IUriValuesDAO} + */ +public class UriValue implements IUriValue { + + private final int uriId; + + /** the freenet uri */ + private FreenetURI uri = null; + /** block count or -1 if its need to be fetched */ + private int blockCount = -1; + /** TODO */ + private Map blocks = new HashMap<>(); + /** TODO */ + private String successSegments = null; + /** TODO */ + private String success = ""; + /** 'MM.yyyy' formated date values values comma separated */ + private String history = ""; + /** TODO */ + private int segment = -1; + + public UriValue(int uriId) { + this.uriId = uriId; + } + + public UriValue(int uriId, FreenetURI uri) { + this(uriId); + this.uri = uri; + } + + @Override + public int getUriId() { + return uriId; + } + + @Override + public FreenetURI getUri() { + return uri; + } + + @Override + public void setUri(FreenetURI uri) { + this.uri = uri; + } + + @Override + public void setBlockCount(int blockCount) { + this.blockCount = blockCount; + } + + @Override + public int getBlockCount() { + return blockCount; + } + + @Override + public Map getBlocks() { + return blocks; + } + + @Override + public void setBlocks(Map blocks) { + this.blocks = blocks; + } + + @Override + public String getSuccessSegments() { + return successSegments; + } + + @Override + public void setSuccessSegments(String successSegments) { + this.successSegments = successSegments; + } + + @Override + public String getSuccess() { + return success; + } + + @Override + public void setSuccess(String success) { + this.success = success; + } + + @Override + public String getHistory() { + return history; + } + + @Override + public void setHistory(String history) { + this.history = history; + } + + @Override + public int getSegment() { + return segment; + } + + @Override + public void setSegment(int segment) { + this.segment = segment; + } + + @Override + public String toString() { + return String.format("UriValue [uriId=%s, uri=%s, blockCount=%s, blocks=%s, successSegments=%s, success=%s, history=%s, segment=%s]", uriId, uri, blockCount, blocks, successSegments, success, + history, segment); + } + + @Override + public int hashCode() { + return Objects.hash(blockCount, blocks, history, segment, success, successSegments, uri, uriId); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if ((obj == null) || (getClass() != obj.getClass())) + return false; + final UriValue other = (UriValue) obj; + return blockCount == other.blockCount && Objects.equals(blocks, other.blocks) && Objects.equals(history, other.history) && segment == other.segment && Objects.equals(success, other.success) + && Objects.equals(successSegments, other.successSegments) && Objects.equals(uri, other.uri) && uriId == other.uriId; + } + +} diff --git a/src/keepalive/urivalues/package-info.java b/src/keepalive/urivalues/package-info.java new file mode 100644 index 0000000..9871a17 --- /dev/null +++ b/src/keepalive/urivalues/package-info.java @@ -0,0 +1,4 @@ +/** + * This package includes all classes and interfaces to save and load the values that KeepAlive uses to reinsert a {@link freenet.keys.FreenetURI} + */ +package keepalive.urivalues; diff --git a/src/keepalive/web/AdminPage.java b/src/keepalive/web/AdminPage.java index c1f57b1..83cfd8b 100644 --- a/src/keepalive/web/AdminPage.java +++ b/src/keepalive/web/AdminPage.java @@ -18,369 +18,343 @@ */ package keepalive.web; -import freenet.keys.FreenetURI; - import java.net.MalformedURLException; import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.List; +import freenet.keys.FreenetURI; import keepalive.Plugin; +import keepalive.exceptions.DAOException; +import keepalive.model.PropertiesKey; +import keepalive.model.SuccessValues; +import keepalive.model.UiKey; +import keepalive.urivalues.IUriValue; import pluginbase.PageBase; public class AdminPage extends PageBase { - - private Plugin plugin; - - private final String formPassword; - - public AdminPage(Plugin plugin, String formPassword) { - super("", "Keep Alive", plugin, true); - this.plugin = plugin; - this.formPassword = formPassword; - addPageToMenu("Start reinsertion of sites", "Add or remove sites you like to reinsert"); - } - - @Override - protected void handleRequest() { - try { - if (formPassword.equals(getParam("formPassword"))) { - // start reinserter - if (getParam("start") != null) { - plugin.startReinserter(getIntParam("start")); - } - - // stop reinserter - if (getParam("stop") != null) { - plugin.stopReinserter(); - } - - // modify power - if (getParam("modify_power") != null) { - setIntPropByParam("power", 1); - saveProp(); - } - - // modify splitfile tolerance - if (getParam("splitfile_tolerance") != null) { - setIntPropByParam("splitfile_tolerance", 0); - saveProp(); - } - - // modify splitfile tolerance - if (getParam("splitfile_test_size") != null) { - setIntPropByParam("splitfile_test_size", 10); - saveProp(); - } - - // modify timeslot to heal single url - if (getParam("single_url_timeslot") != null) { - setIntPropByParam("single_url_timeslot", 1); - saveProp(); - } - - // modify log level - if (getParam("modify_loglevel") != null || getParam("show_log") != null) { - setIntPropByParam("loglevel", 0); - saveProp(); - } - - // clear logs - if (getParam("clear_logs") != null) { - plugin.clearAllLogs(); - } - - // clear history - if (getParam("clear_history") != null) { - removeProp("history_" + getParam("clear_history")); - } - - // add uris - if (getParam("uris") != null) { - addUris(); - } - - // remove uri - if (getParam("remove") != null) { - removeUri(); - } - } - - // boxes - int[] ids = plugin.getIds(); - unsupportedKeysBox(ids); - sitesBox(ids); - logBox(); - configurationBox(); - historyBox(ids); - - // info box - addBox("Information", - html("info", formPassword).replaceAll("#1", plugin.getVersion()), "page-kp-info"); - - } catch (Exception e) { - log("AdminPage.handleRequest(): " + e.getMessage()); - } - } - - private void historyBox(int[] ids) throws Exception { - StringBuilder html = new StringBuilder(""); - for (int id : ids) { - html.append(""); - } - html.append("
") - .append(getShortUri(id)) - .append(""); - - if (getProp("history_" + id) != null) { - html.append(getProp("history_" + id) - .replaceAll("-", "=") - .replaceAll(",", "%, ")) - .append("%"); - } - - html.append("clear
"); - addBox("Lowest rate of blocks availability (monthly)", html.toString(), "page-kp-rate"); - } - - private void configurationBox() throws Exception { - StringBuilder html = new StringBuilder(html("properties", formPassword)); - html = new StringBuilder(html.toString().replaceAll("#1", getProp("power"))); - html = new StringBuilder(html.toString().replaceAll("#2", getProp("loglevel"))); - html = new StringBuilder(html.toString().replaceAll("#3", getProp("splitfile_tolerance"))); - html = new StringBuilder(html.toString().replaceAll("#4", getProp("splitfile_test_size"))); - html = new StringBuilder(html.toString().replaceAll("#5", getProp("single_url_timeslot"))); - addBox("Configuration", html.toString(), "page-kp-config"); - } - - private void logBox() throws Exception { - if (getParam("master_log") != null || getParam("log") != null) { - String log; - if (getParam("master_log") != null) { - log = plugin.getLog(); - } else { - log = plugin.getLog(plugin.getLogFilename(getIntParam("log"))); - } - - if (log == null) { - log = ""; - } - - StringBuilder html = new StringBuilder( - ("" + log + "") - .replaceAll("\n", "
") - .replaceAll(" {2}", "    ")); - - if (getParam("master_log") != null) { - addBox("Master log", html.toString(), null); - } else { - addBox("Log for " + getShortUri(getIntParam("log")), html.toString(), null); - } - } - } - - private void sitesBox(int[] ids) throws Exception { - StringBuilder html = new StringBuilder(html("add_key", formPassword)) - .append("
") - .append("") - .append("") - .append("") - .append("") - .append(""); - - for (int id : ids) { - String uri = getProp("uri_" + id); - int success = plugin.getSuccessValues(id)[0]; - int failure = plugin.getSuccessValues(id)[1]; - - int persistence = 0; - if (success > 0) { - persistence = (int) ((double) success / (success + failure) * 100); - } - - int availableSegments = plugin.getSuccessValues(id)[2]; - int finishedSegmentsCount = getIntProp("segment_" + id) + 1; - - int segmentsAvailability = 0; - if (finishedSegmentsCount > 0) { - segmentsAvailability = (int) ((double) availableSegments / finishedSegmentsCount * 100); - } - - html.append("" + ""); - - if (id == getIntProp("active")) { - html.append(""); - } else { - html.append(""); - } - - html.append(""); - } - - html.append("
URItotal
blocks
available
blocks
missed
blocks
blocks
availability
segments
availability
Actions
") - .append(getShortUri(id)) - .append("") - .append(getProp("blocks_" + id)) - .append("") - .append(success) - .append("") - .append(failure) - .append("") - .append(persistence) - .append(" %") - .append(segmentsAvailability) - .append(" %removelogstopactivestart
"); - addBox("Add or remove a key", html.toString(), "page-kp-keys"); - } - - private void unsupportedKeysBox(int[] ids) throws Exception { - StringBuilder zeroBlockSites = new StringBuilder(); - for (int id : ids) { - if (getProp("blocks_" + id).equals("0")) { - if (zeroBlockSites.length() > 0) { - zeroBlockSites.append("
"); - } - zeroBlockSites.append(getProp("uri_" + id)); - } - } - - if (zeroBlockSites.length() > 0) { - addBox("Unsupported keys", - html("unsupported_keys", formPassword).replaceAll("#", zeroBlockSites.toString()), null); - } - } - - private void addUris() throws Exception { - for (String splitURI : getParam("uris").split("\n")) { - // validate - String uriOrig = URLDecoder.decode(splitURI, "UTF8").trim(); - if (uriOrig.equals("")) { - continue; //ignore blank lines. - } - - String uri = uriOrig; - int begin = uri.indexOf("@") - 3; - if (begin > 0) { - uri = uri.substring(begin); - } - - try { - uri = new FreenetURI(uri).toString(); - - // add if not already on the list - if (!isDuplicate(uri)) { - int[] ids = plugin.getIds(); - - int id; - if (ids.length == 0) { - id = 0; - } else { - id = ids[ids.length - 1] + 1; - } - - setProp("ids", getProp("ids") + id + ","); - setProp("uri_" + id, uri); - setProp("blocks_" + id, "?"); - setProp("success_" + id, ""); - setIntProp("segment_" + id, -1); - } - } catch (MalformedURLException e) { - addBox("URI not valid!", "You have typed:

" + uriOrig, null); - } - } - } - - private void removeUri() throws Exception { - plugin.removeUri(getIntParam("remove")); - } - - // TODO - protected synchronized void updateUskEdition(int siteId) { - try { - - String siteUri = plugin.getProp("uri_" + siteId); - String id = "updateUskEdition" + System.currentTimeMillis(); - fcp.sendClientGet(id, siteUri); - - for (int secs = 0; secs < 300 && getMessage(id, "AllData") == null; secs++) { - wait(1_000); - } - - if (getRedirectURI() != null) { - plugin.setProp("uri_" + siteId, getRedirectURI()); - log("RedirectURI: " + getRedirectURI(), 1); - } - - } catch (Exception e) { - log("AdminPage.updateUskEdition(): " + e.getMessage()); - } - } - - private String getShortUri(int siteId) { - try { - - String uri = getProp("uri_" + siteId); - if (uri.length() > 80) { - return uri.substring(0, 20) + "...." + uri.substring(uri.length() - 50); - } else { - return uri; - } - - } catch (Exception e) { - log("AdminPage.getShortUri(): " + e.getMessage()); - return null; - } - } - - private void setIntPropByParam(String cPropName, int nMinValue) { - try { - - int value = nMinValue; - - try { - value = getIntParam(cPropName); - } catch (Exception ignored) { - } - - if (value != -1 && value < nMinValue) { - value = nMinValue; - } - - setIntProp(cPropName, value); - saveProp(); - - } catch (Exception e) { - log("AdminPage.setPropByParam(): " + e.getMessage()); - } - } - - private boolean isDuplicate(String uri) { - boolean isDuplicate = plugin.isDuplicate(uri); - - if (isDuplicate) { - addBox("Duplicate URI", "We are already keeping this key alive:

" + uri, null); - } - - return isDuplicate; - } + + private final Plugin ownPlugin; + private final String formPassword; + private static final String FORM_PASS = "&formPassword="; + + public AdminPage(Plugin plugin, String formPassword) { + super("", "Keep Alive", plugin, true); + this.ownPlugin = plugin; + this.formPassword = formPassword; + addPageToMenu("Start reinsertion of sites", "Add or remove sites you like to reinsert"); + } + + @Override + protected void handleRequest() { + try { + if (formPassword.equals(getParam(UiKey.FORMPASSWORD))) { + // start reinserter + if (isParamSet(UiKey.START)) { + ownPlugin.startReinserter(getIntParam(UiKey.START)); + } + + // stop reinserter + if (isParamSet(UiKey.STOP)) { + ownPlugin.stopReinserter(); + } + + // modify power + if (isParamSet(PropertiesKey.POWER)) { + setIntPropByParam(PropertiesKey.POWER, 1); + saveProp(); + } + + // modify splitfile tolerance + if (isParamSet(PropertiesKey.SPLITFILE_TOLERANCE)) { + setIntPropByParam(PropertiesKey.SPLITFILE_TOLERANCE, 0); + saveProp(); + } + + // modify splitfile tolerance + if (isParamSet(PropertiesKey.SPLITFILE_TEST_SIZE)) { + setIntPropByParam(PropertiesKey.SPLITFILE_TEST_SIZE, 10); + saveProp(); + } + + // modify timeslot to heal single url + if (isParamSet(PropertiesKey.SINGLE_URL_TIMESLOT)) { + setIntPropByParam(PropertiesKey.SINGLE_URL_TIMESLOT, 1); + saveProp(); + } + + // modify log level + if (isParamSet(UiKey.MODIFY_LOGLEVEL) || isParamSet(UiKey.SHOW_LOG)) { + setIntPropByParam(PropertiesKey.LOGLEVEL, 0); + saveProp(); + } + + // clear logs + if (isParamSet(UiKey.CLEAR_LOGS)) { + ownPlugin.clearAllLogs(); + } + + // clear history + if (isParamSet(UiKey.CLEAR_HISTORY)) { + clearHistory(); + } + + // add uris + if (isParamSet(UiKey.URIS)) { + addUris(); + } + + // remove uri + if (isParamSet(UiKey.REMOVE)) { + removeUri(); + } + + // remove regex + if (isParamSet(UiKey.REMOVE_WITH_REGEX)) { + removeWithRegex(); + } + + // remove all + if (isParamSet(UiKey.REMOVE_ALL)) { + removeAllUris(); + } + } + + // refresh boxes + final List uriValues = ownPlugin.uriPropsDAO.getAll(); + unsupportedKeysBox(uriValues); + sitesBox(uriValues); + logBox(); + configurationBox(); + historyBox(uriValues); + + // info box + addBox("Information", html("info", formPassword).replace("#1", ownPlugin.getVersion()), "page-kp-info"); + + } catch (final Exception e) { + log("AdminPage.handleRequest(): " + e.getMessage()); + } + } + + private void clearHistory() throws Exception { + int value = -1; + try { + value = Integer.parseInt(getParam(UiKey.CLEAR_HISTORY)); + } catch (final NumberFormatException e) {/* ignore */} + + final IUriValue uriValue = ownPlugin.uriPropsDAO.read(value); + if (uriValue != null) { + uriValue.setHistory(null); + ownPlugin.uriPropsDAO.update(uriValue); + } + } + + private void historyBox(List uriValues) { + final StringBuilder html = new StringBuilder(""); + + for (final IUriValue uriValue : uriValues) { + html.append(""); + } + html.append("
") + .append(uriValue.getShortUri()) + .append(""); + + final String history = uriValue.getHistory(); + if (history != null && !history.trim().isEmpty()) { + html.append(history.replace('-', '=').replace(",", "%, ")) + .append("%"); + } + + html.append("clear
"); + + addBox("Lowest rate of blocks availability (monthly)", html.toString(), "page-kp-rate"); + } + + private void configurationBox() throws Exception { + StringBuilder html = new StringBuilder(html("properties", formPassword)); + html = new StringBuilder(html.toString().replace("#1", getProp(PropertiesKey.POWER))); + html = new StringBuilder(html.toString().replace("#2", getProp(PropertiesKey.LOGLEVEL))); + html = new StringBuilder(html.toString().replace("#3", getProp(PropertiesKey.SPLITFILE_TOLERANCE))); + html = new StringBuilder(html.toString().replace("#4", getProp(PropertiesKey.SPLITFILE_TEST_SIZE))); + html = new StringBuilder(html.toString().replace("#5", getProp(PropertiesKey.SINGLE_URL_TIMESLOT))); + addBox("Configuration", html.toString(), "page-kp-config"); + } + + private void logBox() throws Exception { + if (isParamSet(UiKey.MASTER_LOG) || isParamSet(UiKey.LOG)) { + String log; + IUriValue uriValue = null; + if (isParamSet(UiKey.MASTER_LOG)) { + log = ownPlugin.getLog(); + } else { + uriValue = ownPlugin.uriPropsDAO.read(getIntParam(UiKey.LOG)); + log = ownPlugin.getLog(ownPlugin.getLogFilename(uriValue)); + } + + if (log == null) + log = ""; + + final StringBuilder html = new StringBuilder( + ("" + log + "") + .replace("\n", "
") + .replaceAll(" {2}", "    ")); + + if (isParamSet(UiKey.MASTER_LOG)) { + addBox("Master log", html.toString(), "log_anchor"); + } else if (uriValue != null) { + addBox("Log for " + uriValue.getShortUri(), html.toString(), "log_anchor"); + } + } + } + + private void sitesBox(List uriValues) throws Exception { + final StringBuilder html = new StringBuilder(html("add_key", formPassword)).append("
"); + + final StringBuilder htmlEntries = new StringBuilder(); + for (final IUriValue uriValue : uriValues) { + final String uri = uriValue.getUri().toString(); + + final SuccessValues successValues = ownPlugin.getSuccessValues(uriValue); + final int success = successValues.getSuccess(); + final int failure = successValues.getFailed(); + + int persistence = 0; + if (success > 0) { + persistence = (int) ((double) success / (success + failure) * 100); + } + + final int availableSegments = successValues.getAvailableSegments(); + final int finishedSegmentsCount = uriValue.getSegment() + 1; + + int segmentsAvailability = 0; + if (finishedSegmentsCount > 0) { + segmentsAvailability = (int) ((double) availableSegments / finishedSegmentsCount * 100); + } + + final boolean isActive = uriValue.getUriId() == getIntProp(PropertiesKey.ACTIVE); + final String entryHtml = html("url_entry", formPassword) + .replace("${url_full}", uri) + .replace("${url_short}", uriValue.getShortUri()) + .replace("${url_blockSize}", Integer.toString(uriValue.getBlocks().size())) + .replace("${url_success}", Integer.toString(success)) + .replace("${url_failure}", Integer.toString(failure)) + .replace("${url_persistence}", Integer.toString(persistence)) + .replace("${url_segmentsAvailability}", Integer.toString(segmentsAvailability)) + .replace("${url_id}", Integer.toString(uriValue.getUriId())) + .replace("${url_modus}", isActive ? "stop" : "start") + .replace("${url_active}", isActive ? "active" : ""); + htmlEntries.append(entryHtml); + } + + html.append(html("overview_table", formPassword).replace("${url_entries}", htmlEntries.toString())); + + addBox("Add or remove a key", html.toString(), "page-kp-keys"); + } + + private void unsupportedKeysBox(List uriValues) throws Exception { + final StringBuilder zeroBlockSites = new StringBuilder(); + + for (final IUriValue uriValue : uriValues) { + if (uriValue.getBlockCount() == 0) { + if (zeroBlockSites.length() > 0) + zeroBlockSites.append("
"); + + zeroBlockSites.append(uriValue.getUri().toString()); + } + } + + if (zeroBlockSites.length() > 0) { + addBox("Unsupported keys", html("unsupported_keys", formPassword).replace("#", zeroBlockSites.toString()), null); + } + } + + private void addUris() throws Exception { + final String uris = getParam(UiKey.URIS); + if (uris == null || uris.trim().isEmpty()) + return; + + for (final String splitURI : uris.split("\n")) { + // validate + final String uriOrig = URLDecoder.decode(splitURI.trim(), StandardCharsets.UTF_8.toString()).trim(); + if (uriOrig.trim().isEmpty()) + continue; // ignore blank lines. + + try { + final FreenetURI freenetUri = new FreenetURI(uriOrig); + + // add if not already on the list + if (!isDuplicate(freenetUri)) { + ownPlugin.uriPropsDAO.create(freenetUri); + } + } catch (final MalformedURLException e) { + addBox("URI not valid!", "You have typed:

" + uriOrig, null); + } + } + + saveProp(); + } + + private void removeUri() throws Exception { + final IUriValue uriValue = ownPlugin.uriPropsDAO.read(getIntParam(UiKey.REMOVE)); + if (uriValue != null) + ownPlugin.removeUriAndFiles(uriValue); + + ownPlugin.saveProp(true); + } + + private void removeWithRegex() throws Exception { + final String regex = getParam(UiKey.REMOVE_REGEX); + if (regex == null || regex.trim().isEmpty()) { + log("regex value is empty"); + return; + } + + for (final IUriValue uriValue : ownPlugin.uriPropsDAO.getAll()) { + try { + final String uri = uriValue.getUri().toString(); + if (uri.matches(regex)) { + ownPlugin.removeUriAndFiles(uriValue); + } + } catch (final Exception e) { + log("AdminPage.removeWithRegex(): " + e.getMessage()); + } + } + + ownPlugin.saveProp(true); + } + + private void removeAllUris() throws DAOException { + for (final IUriValue uriValue : ownPlugin.uriPropsDAO.getAll()) + ownPlugin.removeUriAndFiles(uriValue); + + ownPlugin.saveProp(true); + } + + private void setIntPropByParam(PropertiesKey cPropName, int nMinValue) { + int value = nMinValue; + + try { + value = getIntParam(cPropName); + } catch (final Exception ex) {/* ignored */} + + if (value != -1 && value < nMinValue) { + value = nMinValue; + } + + setIntProp(cPropName, value); + saveProp(); + } + + private boolean isDuplicate(FreenetURI freenetUri) throws DAOException { + final boolean isDuplicate = ownPlugin.uriPropsDAO.exist(freenetUri); + if (isDuplicate) + addBox("Duplicate URI", "We are already keeping this key alive:

" + freenetUri, null); + + return isDuplicate; + } + } diff --git a/src/pluginbase/FcpCommandBase.java b/src/pluginbase/FcpCommandBase.java index 75aaadc..4f9b49d 100644 --- a/src/pluginbase/FcpCommandBase.java +++ b/src/pluginbase/FcpCommandBase.java @@ -22,126 +22,127 @@ import java.io.InputStream; import java.io.Writer; import java.util.ArrayList; + import pluginbase.de.todesbaum.util.freenet.fcp2.Command; import pluginbase.de.todesbaum.util.freenet.fcp2.Connection; abstract public class FcpCommandBase extends Command { - + private PageBase page; private String strCommandName; private InputStream dataStream; private int nDataLength; private final Connection fcpConnection; private final ArrayList vFields = new ArrayList<>(); - + FcpCommandBase(Connection fcpConnection, PageBase page) { super(null, null); this.fcpConnection = fcpConnection; this.page = page; } - + public FcpCommandBase(Connection fcpConnection) { super(null, null); this.fcpConnection = fcpConnection; } - + @Override public String getCommandName() { return strCommandName; } - + void setCommandName(String strCommandName) { this.strCommandName = strCommandName; } - + private String getIdentifier(String cIdentifier) throws Exception { try { - + return page.getName() + "_" + cIdentifier + "_" + System.currentTimeMillis(); - - } catch (Exception e) { + + } catch (final Exception e) { throw new Exception("FcpCommandBase.getIdentifier(): " + e.getMessage()); } } - + @Override protected boolean hasPayload() { return (dataStream != null); } - + @Override protected InputStream getPayload() { return dataStream; } - + @Override protected long getPayloadLength() { return nDataLength; } - + @Override protected void write(Writer writer) throws IOException { try { - - for (String cField : vFields) { + + for (final String cField : vFields) { writer.write(cField + "\n"); } - - } catch (IOException e) { + + } catch (final IOException e) { page.log("FcpCommandBase.write(): " + e.getMessage(), 1); throw new IOException("FcpCommandBase.write(): " + e.getMessage()); } } - + protected void init(String strCommand, String strIdentifierSuffix) throws Exception { try { - + if (strIdentifierSuffix == null) { strIdentifierSuffix = ""; } vFields.clear(); setCommandName(strCommand); field("Identifier", getIdentifier(strIdentifierSuffix)); - - } catch (Exception e) { + + } catch (final Exception e) { throw new Exception("FcpCommandBase.init(): " + e.getMessage()); } } - + protected void field(String strKey, String strValue) throws Exception { try { - + vFields.add(strKey + "=" + strValue); - - } catch (Exception e) { + + } catch (final Exception e) { throw new Exception("FcpCommandBase.field(): " + e.getMessage()); } } - + protected void field(String strKey, int nValue) throws Exception { try { - + vFields.add(strKey + "=" + nValue); - - } catch (Exception e) { + + } catch (final Exception e) { throw new Exception("FcpCommandBase.field(): " + e.getMessage()); } } - + protected void send(InputStream dataStream, int nDataLength) throws Exception { try { - + this.dataStream = dataStream; this.nDataLength = nDataLength; fcpConnection.execute(this); - + } catch (IllegalStateException | IOException e) { throw new Exception("FcpCommandBase.send(): " + e.getMessage()); } } - + protected void send() throws Exception { send(null, 0); } - + } diff --git a/src/pluginbase/FcpCommands.java b/src/pluginbase/FcpCommands.java index ba4fdec..da67072 100644 --- a/src/pluginbase/FcpCommands.java +++ b/src/pluginbase/FcpCommands.java @@ -18,65 +18,65 @@ */ package pluginbase; -import pluginbase.de.todesbaum.util.freenet.fcp2.Connection; +import static java.nio.charset.StandardCharsets.UTF_8; import java.io.ByteArrayInputStream; import java.io.InputStream; -import static java.nio.charset.StandardCharsets.UTF_8; +import pluginbase.de.todesbaum.util.freenet.fcp2.Connection; public class FcpCommands extends FcpCommandBase { - + FcpCommands(Connection fcpConnection, PageBase page) { super(fcpConnection, page); } - + public void sendGenerateSSK(String strId) throws Exception { try { - + init("GenerateSSK", strId); send(); - - } catch (Exception e) { + + } catch (final Exception e) { throw new Exception("FcpCommand.sendGenerateSSK(): " + e.getMessage()); } } - + public void sendClientGet(String strId, String strUri) throws Exception { try { - + init("ClientGet", strId); field("URI", strUri); send(); - - } catch (Exception e) { + + } catch (final Exception e) { throw new Exception("FcpCommand.sendClientGet(): " + e.getMessage()); } } - + public void sendClientPut(String strId, String strUri, InputStream dataStream, int nDataLength) throws Exception { try { - + init("ClientPut", strId); field("URI", strUri); field("DataLength", nDataLength); send(dataStream, nDataLength); - - } catch (Exception e) { + + } catch (final Exception e) { throw new Exception("FcpCommand.sendClientPut(): " + e.getMessage()); } } - + public void sendClientPut(String cId, String cUri, String cContent) throws Exception { try { - - byte[] aContent = cContent.getBytes(UTF_8); - ByteArrayInputStream dataStream = new ByteArrayInputStream(aContent); + + final byte[] aContent = cContent.getBytes(UTF_8); + final ByteArrayInputStream dataStream = new ByteArrayInputStream(aContent); sendClientPut(cId, cUri, dataStream, aContent.length); - - } catch (Exception e) { + + } catch (final Exception e) { throw new Exception("FcpCommand.sendClientPut(): " + e.getMessage()); } } - + } diff --git a/src/pluginbase/PageBase.java b/src/pluginbase/PageBase.java index b5622c3..924fb9d 100644 --- a/src/pluginbase/PageBase.java +++ b/src/pluginbase/PageBase.java @@ -18,77 +18,82 @@ */ package pluginbase; -import freenet.l10n.NodeL10n; -import pluginbase.de.todesbaum.util.freenet.fcp2.Message; -import freenet.clients.http.InfoboxNode; -import freenet.clients.http.PageNode; -import freenet.clients.http.Toadlet; -import freenet.clients.http.ToadletContext; -import freenet.clients.http.ToadletContextClosedException; -import freenet.l10n.BaseL10n.LANGUAGE; -import freenet.pluginmanager.*; -import freenet.support.api.HTTPRequest; -import freenet.support.HTMLNode; +import static java.nio.charset.StandardCharsets.UTF_8; import java.io.ByteArrayOutputStream; -import java.io.InputStream; import java.io.IOException; +import java.io.InputStream; import java.net.URI; import java.net.URLConnection; -import java.util.TreeMap; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; -import static java.nio.charset.StandardCharsets.*; - -abstract public class PageBase extends Toadlet implements FredPluginL10n { +import freenet.clients.http.InfoboxNode; +import freenet.clients.http.PageNode; +import freenet.clients.http.Toadlet; +import freenet.clients.http.ToadletContext; +import freenet.clients.http.ToadletContextClosedException; +import freenet.l10n.BaseL10n.LANGUAGE; +import freenet.l10n.NodeL10n; +import freenet.pluginmanager.FredPluginL10n; +import freenet.support.HTMLNode; +import freenet.support.api.HTTPRequest; +import keepalive.model.PropertiesKey; +import keepalive.model.UiKey; +import pluginbase.de.todesbaum.util.freenet.fcp2.Message; +public abstract class PageBase extends Toadlet implements FredPluginL10n { + public PluginBase plugin; - + protected FcpCommands fcp; - + private PageNode page; - private ArrayList vBoxes = new ArrayList(); - private TreeMap mMessages = new TreeMap<>(); + private final List vBoxes = new ArrayList<>(); + private final Map mMessages = new TreeMap<>(); private String strPageName; private String strPageTitle; private String strRefreshTarget; private int nRefreshPeriod = -1; private URI uri; private HTTPRequest httpRequest; - private TreeMap mRedirectURIs = new TreeMap<>(); + private final TreeMap mRedirectURIs = new TreeMap<>(); private String strRedirectURI; private boolean bFullAccessHostsOnly; - - public PageBase(String cPageName, String cPageTitle, PluginBase plugin, boolean bFullAccessHostsOnly) { + + protected PageBase(String cPageName, String cPageTitle, PluginBase plugin, boolean bFullAccessHostsOnly) { super(plugin.pluginContext.node.clientCore.makeClient((short) 3, false, false)); - + try { this.strPageName = cPageName; this.strPageTitle = cPageTitle; this.plugin = plugin; this.bFullAccessHostsOnly = bFullAccessHostsOnly; - + // register this page and add to menu plugin.webInterface.registerInvisible(this); plugin.log("page '" + cPageName + "' registered"); - + // fcp request object fcp = new FcpCommands(plugin.fcpConnection, this); - - } catch (Exception e) { + + } catch (final Exception e) { plugin.log("PageBase(): " + e.getMessage(), 1); } } - + public String getName() { return strPageName; } - + @Override public String path() { return plugin.getPath() + "/" + strPageName; } - + /** * * @param cKey @@ -98,7 +103,7 @@ public String path() { public String getString(String cKey) { // FredPluginL10n return plugin.getString(cKey); } - + /** * * @param newLanguage @@ -107,11 +112,11 @@ public String getString(String cKey) { // FredPluginL10n public void setLanguage(LANGUAGE newLanguage) { // FredPluginL10n plugin.setLanguage(newLanguage); } - + @Override public void handleMethodGET(URI uri, HTTPRequest request, ToadletContext ctx) throws ToadletContextClosedException, IOException { try { - + vBoxes.clear(); if (!bFullAccessHostsOnly || ctx.isAllowedFullAccess()) { this.uri = uri; @@ -120,63 +125,63 @@ public void handleMethodGET(URI uri, HTTPRequest request, ToadletContext ctx) th } else { addBox("Access denied!", "Access to this page for hosts with full access rights only.", null); } - + makePage(uri, ctx); - - } catch (Exception e) { + + } catch (final Exception e) { log("PageBase.handleMethodGET(): " + e.getMessage(), 1); } } - + public void handleMethodPOST(URI uri, HTTPRequest request, ToadletContext ctx) throws ToadletContextClosedException, IOException { handleMethodGET(uri, request, ctx); } - + @Override public boolean allowPOSTWithoutPassword() { return true; } - + private String getIdentifier(Message message) throws Exception { try { - - String[] aIdentifier = message.getIdentifier().split("_"); - String cIdentifier = aIdentifier[1]; + + final String[] aIdentifier = message.getIdentifier().split("_"); + final StringBuilder cIdentifier = new StringBuilder().append(aIdentifier[1]); for (int i = 2; i < aIdentifier.length - 1; i++) { - cIdentifier += "_" + aIdentifier[i]; + cIdentifier.append("_").append(aIdentifier[i]); } - return cIdentifier; - - } catch (Exception e) { + return cIdentifier.toString(); + + } catch (final Exception e) { throw new Exception("PageBase.getIdentifier(): " + e.getMessage()); } } - + void addMessage(Message message) throws Exception { try { - + // existing message with same id becomes replaced (e.g. AllData replaces DataFound) mMessages.put(getIdentifier(message), message); - - } catch (Exception e) { + + } catch (final Exception e) { throw new Exception("PageBase.addMessage(): " + e.getMessage()); } } - + void updateRedirectUri(Message message) throws Exception { try { - + // existing RedirectUri with same id becomes replaced mRedirectURIs.put(getIdentifier(message), message.get("RedirectUri")); - - } catch (Exception e) { + + } catch (final Exception e) { throw new Exception("PageBase.updateRedirectUri(): " + e.getMessage()); } } - + private void makePage(URI uri, ToadletContext ctx) throws Exception { try { - + String path = uri.getPath().substring(plugin.getPath().length()); if (path.startsWith("/static/")) { path = "/resources" + path; @@ -188,23 +193,23 @@ private void makePage(URI uri, ToadletContext ctx) throws Exception { NodeL10n.getBase().getString("StaticToadlet.pathNotFound")); return; } - - String mimeType = URLConnection.guessContentTypeFromStream(inputStream); - - ByteArrayOutputStream content = new ByteArrayOutputStream(); + + final String mimeType = URLConnection.guessContentTypeFromStream(inputStream); + + final ByteArrayOutputStream content = new ByteArrayOutputStream(); int len; - byte[] contentBytes = new byte[1024]; + final byte[] contentBytes = new byte[1024]; while ((len = inputStream.read(contentBytes)) != -1) { content.write(contentBytes, 0, len); } - + writeReply(ctx, 200, mimeType, "", content.toByteArray(), 0, content.size()); return; } } - + page = plugin.pagemaker.getPageNode(strPageTitle, ctx); - + // refresh page if (nRefreshPeriod != -1) { if (strRefreshTarget == null) { @@ -213,27 +218,27 @@ private void makePage(URI uri, ToadletContext ctx) throws Exception { strRefreshTarget += "?" + uri.getQuery(); } } - page.headNode.addChild("meta", new String[]{"http-equiv", "content"}, - new String[]{"refresh", nRefreshPeriod + ";URL=" + strRefreshTarget}); + page.headNode.addChild("meta", new String[] { "http-equiv", "content" }, + new String[] { "refresh", nRefreshPeriod + ";URL=" + strRefreshTarget }); } - + page.headNode.addChild("link", - new String[]{"rel", "href", "type"}, - new String[]{"stylesheet", "static/style.css", "text/css"}); - + new String[] { "rel", "href", "type" }, + new String[] { "stylesheet", "static/style.css", "text/css" }); + // boxes - for (HTMLNode box : vBoxes) { + for (final HTMLNode box : vBoxes) { page.content.addChild(box); } - + // write writeHTMLReply(ctx, 200, "OK", page.outer.generate()); - + } catch (ToadletContextClosedException | IOException e) { throw new Exception("PageBase.html(): " + e.getMessage()); } } - + // ******************************************** // methods to use in the derived plugin class: // ******************************************** @@ -241,119 +246,149 @@ private void makePage(URI uri, ToadletContext ctx) throws Exception { public void log(String strText, int nLogLevel) { plugin.log(strText, nLogLevel); } - + public void log(String strText) { plugin.log(strText); } - + + public void log(String strText, Object... args) { + plugin.logF(strText, args); + } + // methods to add this page to the plugins' menu (fproxy) protected void addPageToMenu(String strMenuTitle, String strMenuTooltip) { try { - + plugin.pluginContext.pluginRespirator.getToadletContainer().unregister(this); plugin.pluginContext.pluginRespirator.getToadletContainer().register(this, plugin.getCategory(), this.path(), true, strMenuTitle, strMenuTooltip, bFullAccessHostsOnly, null); log("page '" + strPageName + "' added to menu"); - - } catch (Exception e) { + + } catch (final Exception e) { log("PageBase.addPageToMenu(): " + e.getMessage(), 1); } } - + // methods to build the page protected void addBox(String title, String htmlBody, String id) { try { - - InfoboxNode box = plugin.pagemaker.getInfobox(title); + final InfoboxNode box = plugin.pagemaker.getInfobox(title); if (id != null) { box.outer.addAttribute("id", id); } - htmlBody = htmlBody.replaceAll("'", "\""); + htmlBody = htmlBody.replace('\'', '"'); box.content.addChild("%", htmlBody); vBoxes.add(box.outer); - - } catch (Exception e) { + + } catch (final Exception e) { log("PluginBase.addBox(): " + e.getMessage(), 1); } } - + protected String html(String name, String formPassword) throws Exception { - try (InputStream stream = - getClass() - .getResourceAsStream("/resources/templates/" + name + ".html")) { - - ByteArrayOutputStream content = new ByteArrayOutputStream(); + try (InputStream stream = getClass() + .getResourceAsStream("/resources/templates/" + name + ".html")) { + + final ByteArrayOutputStream content = new ByteArrayOutputStream(); int len; - byte[] contentBytes = new byte[1024]; + final byte[] contentBytes = new byte[1024]; while ((len = stream.read(contentBytes)) != -1) { content.write(contentBytes, 0, len); } - - return content.toString(UTF_8.name()).replaceAll("\\$\\{formPassword}", formPassword); - } catch (IOException e) { + + return content.toString(StandardCharsets.UTF_8.toString()).replace("${formPassword}", formPassword); + } catch (final IOException e) { throw new Exception("PageBase.html(): " + e.getMessage()); } } - + // methods to make the page refresh protected void setRefresh(int nPeriod, String strTarget) { this.nRefreshPeriod = nPeriod; this.strRefreshTarget = strTarget; } - + protected void setRefresh(int nPeriod) { setRefresh(nPeriod, null); } - + // methods to handle http requests (both get and post) abstract protected void handleRequest(); - + protected String getQuery() throws Exception { try { - + return uri.getQuery(); - - } catch (Exception e) { + + } catch (final Exception e) { throw new Exception("PageBase.getQuery(): " + e.getMessage()); } } - + protected HTTPRequest getRequest() { return httpRequest; } - + + protected boolean isParamSet(UiKey propKey) throws Exception { + return getParam(propKey.toString()) != null; + } + + protected boolean isParamSet(PropertiesKey propKey) throws Exception { + return getParam(propKey.toString()) != null; + } + + protected String getParam(UiKey propKey) throws Exception { + return getParam(propKey.toString()); + } + protected String getParam(String strKey) throws Exception { + if (strKey == null) + return null; + try { - - if (httpRequest.getMethod().toUpperCase().equals("GET") && !httpRequest.getParam(strKey).equals("")) { + if ("GET".equalsIgnoreCase(httpRequest.getMethod()) && !"".equals(httpRequest.getParam(strKey))) { return httpRequest.getParam(strKey); - } else if (httpRequest.getMethod().toUpperCase().equals("POST") && httpRequest.getPart(strKey) != null) { - byte[] aContent = new byte[(int) httpRequest.getPart(strKey).size()]; + } + if ("POST".equalsIgnoreCase(httpRequest.getMethod()) && httpRequest.getPart(strKey) != null) { + final byte[] aContent = new byte[(int) httpRequest.getPart(strKey).size()]; httpRequest.getPart(strKey).getInputStream().read(aContent); return new String(aContent, UTF_8); - } else { - return null; } - - } catch (IOException e) { + + return null; + } catch (final IOException e) { throw new Exception("PageBase.getParam(): " + e.getMessage()); } } - - protected int getIntParam(String cKey) throws Exception { + + protected int getIntParam(UiKey key) throws Exception { try { - - return Integer.valueOf(getParam(cKey)); - - } catch (Exception e) { + return Integer.parseInt(getParam(key)); + } catch (final Exception e) { throw new Exception("PageBase.getIntParam(): " + e.getMessage()); } } - + + protected int getIntParam(PropertiesKey key) throws Exception { + try { + return Integer.parseInt(getParam(key.toString())); + } catch (final Exception e) { + throw new Exception("PageBase.getIntParam(): " + e.getMessage()); + } + } + + protected int getIntParam2(String key) throws Exception { + try { + return Integer.parseInt(getParam(key)); + } catch (final Exception e) { + throw new Exception("PageBase.getIntParam(): " + e.getMessage()); + } + } + // methods to handle fcp messages protected Message getMessage(String cId, String cMessageType) throws Exception { try { - + Message message = mMessages.get(cId); if (message != null && !message.getName().equals(cMessageType)) { message = null; @@ -364,60 +399,59 @@ protected Message getMessage(String cId, String cMessageType) throws Exception { mRedirectURIs.remove(cId); } return message; // returns null if no message - - } catch (Exception e) { + + } catch (final Exception e) { throw new Exception("PageBase.getMessage(): " + e.getMessage()); } } - + protected String getRedirectURI() { return strRedirectURI; } - + protected String[] getSSKKeypair(String cId) { try { - - Message message = getMessage(cId, "SSKKeypair"); + + final Message message = getMessage(cId, "SSKKeypair"); if (message != null) { - return new String[]{message.get("InsertURI"), message.get("RequestURI")}; - } else { - return null; + return new String[] { message.get("InsertURI"), message.get("RequestURI") }; } - - } catch (Exception e) { + return null; + + } catch (final Exception e) { log("PageBase.getSSKKeypair(): " + e.getMessage(), 1); return null; } } - + // methods to set and get persistent properties - public void saveProp() { + protected void saveProp() { plugin.saveProp(); } - - public void setProp(String cKey, String cValue) throws Exception { - plugin.setProp(cKey, cValue); + + protected void setProp(PropertiesKey key, String value) { + plugin.setProp(key, value); } - - public String getProp(String cKey) throws Exception { - return plugin.getProp(cKey); + + protected String getProp(PropertiesKey key) { + return plugin.getProp(key); } - - public void setIntProp(String cKey, int nValue) throws Exception { - plugin.setIntProp(cKey, nValue); + + protected void setIntProp(PropertiesKey key, int nValue) { + plugin.setIntProp(key, nValue); } - - public int getIntProp(String cKey) throws Exception { - return plugin.getIntProp(cKey); + + protected int getIntProp(PropertiesKey key) { + return plugin.getIntProp(key); } - - public void removeProp(String cKey) { - plugin.removeProp(cKey); + + protected void removeProp(PropertiesKey key) { + plugin.removeProp(key); } - + // method to allow access to this page for full-access-hosts only public void restrictToFullAccessHosts(boolean bRestrict) { bFullAccessHostsOnly = bRestrict; } - + } diff --git a/src/pluginbase/PluginBase.java b/src/pluginbase/PluginBase.java index d333876..bd5ff66 100644 --- a/src/pluginbase/PluginBase.java +++ b/src/pluginbase/PluginBase.java @@ -18,40 +18,46 @@ */ package pluginbase; -import pluginbase.de.todesbaum.util.freenet.fcp2.Connection; -import pluginbase.de.todesbaum.util.freenet.fcp2.ConnectionListener; -import pluginbase.de.todesbaum.util.freenet.fcp2.Message; -import pluginbase.de.todesbaum.util.freenet.fcp2.Node; -import freenet.clients.http.PageMaker; -import freenet.l10n.BaseL10n.LANGUAGE; -import freenet.pluginmanager.FredPlugin; -import freenet.pluginmanager.FredPluginL10n; -import freenet.pluginmanager.FredPluginThreadless; -import freenet.pluginmanager.FredPluginVersioned; -import freenet.pluginmanager.PluginRespirator; -import freenet.support.plugins.helpers1.PluginContext; -import freenet.support.plugins.helpers1.WebInterface; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.io.PrintWriter; import java.io.RandomAccessFile; +import java.io.StringWriter; +import java.nio.file.Files; import java.text.SimpleDateFormat; import java.util.Date; +import java.util.Map; import java.util.Properties; import java.util.TimeZone; import java.util.TreeMap; -abstract public class PluginBase implements FredPlugin, FredPluginThreadless, - FredPluginVersioned, FredPluginL10n, ConnectionListener { +import freenet.clients.http.PageMaker; +import freenet.l10n.BaseL10n.LANGUAGE; +import freenet.pluginmanager.FredPlugin; +import freenet.pluginmanager.FredPluginL10n; +import freenet.pluginmanager.FredPluginThreadless; +import freenet.pluginmanager.FredPluginVersioned; +import freenet.pluginmanager.PluginRespirator; +import freenet.support.plugins.helpers1.PluginContext; +import freenet.support.plugins.helpers1.WebInterface; +import keepalive.model.PropertiesKey; +import pluginbase.de.todesbaum.util.freenet.fcp2.Connection; +import pluginbase.de.todesbaum.util.freenet.fcp2.ConnectionListener; +import pluginbase.de.todesbaum.util.freenet.fcp2.Message; +import pluginbase.de.todesbaum.util.freenet.fcp2.Node; +public abstract class PluginBase implements FredPlugin, FredPluginThreadless, + FredPluginVersioned, FredPluginL10n, ConnectionListener { + public PluginContext pluginContext; - + PageMaker pagemaker; WebInterface webInterface; Connection fcpConnection; LANGUAGE nodeLanguage; - + private Properties prop; private String strTitle; private String strPath; @@ -59,76 +65,74 @@ abstract public class PluginBase implements FredPlugin, FredPluginThreadless, private String strMenuTitle = null; private String strMenuTooltip = null; private String strVersion = "0.0"; - private SimpleDateFormat dateFormat; - private TreeMap mPages = new TreeMap(); - private TreeMap mLogFiles = new TreeMap<>(); - - public PluginBase(String strPath, String strTitle, String strPropFilename) { + private final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy.MM.dd_HH.mm_ss"); + private final Map mPages = new TreeMap<>(); + private final Map mLogFiles = new TreeMap<>(); + private final boolean stackTrace = true;// FIXME "true".equals(getProp(PropertiesKey.STACKTRACE)); + + protected PluginBase(String strPath, String strTitle, String strPropFilename) { try { - this.strPath = strPath; this.strTitle = strTitle; this.strPropFilename = strPropFilename; - + // prepare and clear log file (new File(strPath)).mkdir(); initLog("log.txt"); - dateFormat = new SimpleDateFormat("yyyy.MM.dd_HH.mm_ss"); + dateFormat.setTimeZone(TimeZone.getDefault()); - + // load properties loadProp(); - if (getProp("loglevel") == null) { - setIntProp("loglevel", 0); + if (getProp(PropertiesKey.LOGLEVEL) == null) { + setIntProp(PropertiesKey.LOGLEVEL, 0); } - - } catch (Exception e) { - log("PluginBase(): " + e.getMessage(), 1); + + } catch (final Exception e) { + log("PluginBase(): " + e.getMessage(), e); } } - + String getCategory() { - return strTitle.replaceAll(" ", "_"); + return strTitle.replace(' ', '_'); } - + String getPath() { return "/" + strPath; } - + public String getPluginDirectory() { return strPath + "/"; } - + @Override public void runPlugin(PluginRespirator pr) { // FredPlugin try { - log(getVersion() + " started"); - + // init plugin context pagemaker = pr.getPageMaker(); pluginContext = new PluginContext(pr); webInterface = new WebInterface(pluginContext); - + // fcp connection connectFcp(); - + // add menu pagemaker.removeNavigationCategory(getCategory()); if (strMenuTitle != null) { webInterface.addNavigationCategory(getPath() + "/", getCategory(), strMenuTooltip, this); } - - } catch (Exception e) { + } catch (final Exception e) { log("PluginBase.runPlugin(): " + e.getMessage(), 1); } } - + @Override public String getVersion() { // FredPluginVersioned - return strTitle + " " + strVersion; + return strVersion; } - + /** * * @param strKey @@ -138,7 +142,7 @@ public String getVersion() { // FredPluginVersioned public String getString(String strKey) { // FredPluginL10n return strKey; } - + /** * * @param language @@ -147,11 +151,11 @@ public String getString(String strKey) { // FredPluginL10n public void setLanguage(LANGUAGE language) { // FredPluginL10n nodeLanguage = language; } - + @Override public void terminate() { // FredPlugin try { - + log("plugin base terminates"); saveProp(); fcpConnection.disconnect(); @@ -160,152 +164,160 @@ public void terminate() { // FredPlugin webInterface = null; pagemaker.removeNavigationCategory(getCategory()); log("plugin base terminated"); - for (RandomAccessFile file : mLogFiles.values()) { + for (final RandomAccessFile file : mLogFiles.values()) { file.close(); } - - } catch (IOException e) { + + } catch (final IOException e) { log("PluginBase.terminate(): " + e.getMessage(), 1); } } - + @Override public void messageReceived(Connection connection, Message message) { // ConnectionListener try { - // errors - if (message.getName().equals("ProtocolError")) + if ("ProtocolError".equals(message.getName())) { log("ProtocolError: " + message.get("CodeDescription")); - else if (message.getName().equals("IdentifierCollision")) + } else if ("IdentifierCollision".equals(message.getName())) { log("IdentifierCollision"); - - // redirect deprecated usk edition - else if (message.getName().equals("GetFailed") && - (message.get("RedirectUri") != null) && !message.get("RedirectUri").equals("")) { + } else if ("GetFailed".equals(message.getName()) && + (message.get("RedirectUri") != null) && !"".equals(message.get("RedirectUri"))) { + // redirect deprecated usk edition log("USK redirected (" + message.getIdentifier() + ")"); - + // reg new edition - String pageName = message.getIdentifier().split("_")[0]; - PageBase page = (PageBase) mPages.get(pageName); + final String pageName = message.getIdentifier().split("_")[0]; + final PageBase page = mPages.get(pageName); if (page != null) page.updateRedirectUri(message); - + // redirect - FcpCommands fcpCommand = new FcpCommands(fcpConnection, null); + final FcpCommands fcpCommand = new FcpCommands(fcpConnection, null); fcpCommand.setCommandName("ClientGet"); fcpCommand.field("Identifier", message.getIdentifier()); fcpCommand.field("URI", message.get("RedirectUri")); fcpCommand.send(); - } - - // register message - else { + } else { + // register message log("fcp: " + message.getName() + " (" + message.getIdentifier() + ")"); - String pageName = message.getIdentifier().split("_")[0]; - PageBase page = (PageBase) mPages.get(pageName); + final String pageName = message.getIdentifier().split("_")[0]; + final PageBase page = mPages.get(pageName); if (page != null) page.addMessage(message); } - - } catch (Exception e) { + + } catch (final Exception e) { log("PluginBase.messageReceived(): " + e.getMessage(), 1); } } - + @Override public void connectionTerminated(Connection connection) { // ConnectionListener log("fcp connection terminated"); } - + private synchronized void connectFcp() { try { - + if (fcpConnection == null || !fcpConnection.isConnected()) { fcpConnection = new Connection(new Node("localhost"), "connection_" + System.currentTimeMillis()); fcpConnection.addConnectionListener(this); fcpConnection.connect(); } - - } catch (IOException e) { + + } catch (final IOException e) { log("PluginBase.connectFcp(): " + e.getMessage(), 1); } } - + private void loadProp() { try { - if (strPropFilename != null) { prop = new Properties(); - File file = new File(strPath + "/" + strPropFilename); - File oldFile = new File(strPath + "/" + strPropFilename + ".old"); - - FileInputStream is; - //always load from the backup if it exists, it is (almost?) - //guaranteed to be good. - if (oldFile.exists()) { - is = new FileInputStream(oldFile); - } else if (file.exists()) { - is = new FileInputStream(file); - } else { - throw new Exception("No Prop file found."); + final File file = new File(strPath, strPropFilename); + final File oldFile = new File(strPath, strPropFilename + ".old"); + + FileInputStream is = null; + try { + //always load from the backup if it exists, it is (almost?) + //guaranteed to be good. + if (oldFile.exists()) { + is = new FileInputStream(oldFile); + } else if (file.exists()) { + is = new FileInputStream(file); + } else { + throw new Exception("No Prop file found."); + } + prop.load(is); + } finally { + if (is != null) + is.close(); } - prop.load(is); - is.close(); } - - } catch (Exception e) { + } catch (final Exception e) { log("PluginBase.loadProp(): " + e.getMessage(), 1); } } - + // ****************************************** // methods to use in the derived page class: // ****************************************** // log files private synchronized void initLog(String strFilename) { try { - if (!mLogFiles.containsKey(strFilename)) { - RandomAccessFile file = new RandomAccessFile(strPath + "/" + strFilename, "rw"); + final RandomAccessFile file = new RandomAccessFile(strPath + "/" + strFilename, "rw"); file.seek(file.length()); mLogFiles.put(strFilename, file); } - - } catch (IOException e) { - log("PluginBase.initLog(): " + e.getMessage()); + } catch (final IOException e) { + log("PluginBase.initLog() - file: %s", e, strFilename); } } - - public synchronized void log(String strFilename, String cText, int nLogLevel) { + + protected synchronized void removeLogFromMap(String strFilename) { try { - - if (nLogLevel <= getIntProp("loglevel")) { + final RandomAccessFile fileHandle = mLogFiles.get(strFilename); + if (fileHandle != null) { + fileHandle.close(); + mLogFiles.remove(strFilename); + } + } catch (final Exception e) { + log("PluginBase.removeLogFromMap() - file: %s", e, strFilename); + } + } + + public synchronized void logFile(String strFilename, String cText, int nLogLevel) { + try { + + if (nLogLevel <= getIntProp(PropertiesKey.LOGLEVEL)) { initLog(strFilename); mLogFiles.get(strFilename).writeBytes(dateFormat.format(new Date()) + " " + cText + "\n"); } - - } catch (Exception e) { - if (!strFilename.equals("log.txt")) // to avoid infinite loop when log.txt was closed on shutdown + + } catch (final Exception e) { + if (!"log.txt".equals(strFilename)) // to avoid infinite loop when log.txt was closed on shutdown { - log("PluginBase.log(): " + e.getMessage()); + log("PluginBase.log():", e); } } } - - public void log(String strFilename, String strText) { - log(strFilename, strText, 0); + + public void logFile(String strFilename, String strText) { + logFile(strFilename, strText, 0); } - + public synchronized String getLog(String filename) { try { - + initLog(filename); - RandomAccessFile file = mLogFiles.get(filename); - int MAX_LOG_LENGTH = 2_000_000; // around 10k lines - StringBuilder buffer = new StringBuilder(); - long fileLength = file.length(); + final RandomAccessFile file = mLogFiles.get(filename); + final int MAX_LOG_LENGTH = 2_000_000; // around 10k lines + final StringBuilder buffer = new StringBuilder(); + final long fileLength = file.length(); if (fileLength > MAX_LOG_LENGTH) { - long skip = fileLength - MAX_LOG_LENGTH; + final long skip = fileLength - MAX_LOG_LENGTH; file.seek(skip); file.readLine(); buffer.append("log contains ").append(skip).append(" preceding bytes (~").append(skip / 200).append(" lines)").append("\n"); @@ -317,192 +329,256 @@ public synchronized String getLog(String filename) { buffer.append(line).append("\n"); } return buffer.toString(); - - } catch (IOException e) { + + } catch (final IOException e) { log("PluginBase.getLog(): " + e.getMessage()); return null; } } - + public void clearLog(String strFilename) { try { - + initLog(strFilename); mLogFiles.get(strFilename).setLength(0); - - } catch (IOException e) { + + } catch (final IOException e) { log("PluginBase.clearLog(): " + e.getMessage()); } } - + public void clearAllLogs() { try { - - for (RandomAccessFile file : mLogFiles.values()) { + + for (final RandomAccessFile file : mLogFiles.values()) { file.setLength(0); } - - } catch (IOException e) { + + } catch (final IOException e) { log("PluginBase.clearAllLogs(): " + e.getMessage()); } } - + public void setLogLevel(int nLevel) throws Exception { try { - - setIntProp("loglevel", nLevel); - - } catch (Exception e) { - throw new Exception("PluginBase.setLogLevel(): " + e.getMessage()); + + setIntProp(PropertiesKey.LOGLEVEL, nLevel); + + } catch (final Exception e) { + throw new Exception("PluginBase.setLogLevel(): " + e.getMessage(), e); } } - + // standard log public void log(String cText) { - log("log.txt", cText, 0); + logFile("log.txt", cText, 0); } - + + public void logF(String cText, Object... args) { + logFile("log.txt", String.format(cText, args), 0); + } + public void log(String cText, int nLogLevel) { - log("log.txt", cText, nLogLevel); + logFile("log.txt", cText, nLogLevel); + } + + public void log(String info, Throwable e) { + final StringBuilder message = new StringBuilder(String.format("%s: %s %s", info, e.getClass().getName(), e.getMessage())); + if (stackTrace) + message.append(System.lineSeparator()).append(stackTraceToString(e)); + + log(message.toString()); + } + + public void log(String info, Throwable e, Object... args) { + log(String.format(info, args), e); + } + + private String stackTraceToString(Throwable e) { + try (final StringWriter sw = new StringWriter(); + final PrintWriter pw = new PrintWriter(sw)) { + e.printStackTrace(pw); + return sw.toString(); + } catch (final IOException e1) { + return String.format("Error in stackTraceToString: %s", e1.getMessage()); + } } - + protected void clearLog() { clearLog("log.txt"); } - + public String getLog() { return getLog("log.txt"); } - + // methods to set the version of the plugin protected void setVersion(String strVersion) { this.strVersion = strVersion; } - + // methods to add the plugin to the nodes' main menu (fproxy) protected void addPluginToMenu(String strMenuTitle, String strMenuTooltip) { this.strMenuTitle = strMenuTitle; this.strMenuTooltip = strMenuTooltip; } - + // methods to add menu items to the plugins menu (fproxy) protected void addMenuItem(String strTitle, String strTooltip, String strUri, boolean isFullAccessHostsOnly) { try { - + pagemaker.addNavigationLink(getCategory(), strUri, strTitle, strTooltip, isFullAccessHostsOnly, null, this); log("item '" + strTitle + "' added to menu"); - - } catch (Exception e) { + + } catch (final Exception e) { log("PluginBase.addMenuItem(): " + e.getMessage()); } } - + // methods to build the plugin protected void addPage(PageBase page) { try { - + mPages.put(page.getName(), page); - - } catch (Exception e) { + + } catch (final Exception e) { log("PluginBase.addPage(): " + e.getMessage()); } } - + // methods to set and get persistent properties public synchronized void saveProp() { + if (prop == null) + return; + try { - - if (prop != null) { - File file = new File(strPath + "/" + strPropFilename); - File oldFile = new File(strPath + "/" + strPropFilename + ".old"); - File newFile = new File(strPath + "/" + strPropFilename + ".new"); - - if (newFile.exists()) { - newFile.delete(); - } - - try (FileOutputStream stream = new FileOutputStream(newFile)) { - prop.store(stream, strTitle); - stream.flush(); - } - - if (oldFile.exists()) { - oldFile.delete(); - } - - if (file.exists()) { - file.renameTo(oldFile); - } - - newFile.renameTo(file); + final File file = new File(strPath, strPropFilename); + final File oldFile = new File(strPath, strPropFilename + ".old"); + final File newFile = new File(strPath, strPropFilename + ".new"); + + if (newFile.exists()) { + Files.delete(newFile.toPath()); } - - } catch (IOException e) { + + try (FileOutputStream stream = new FileOutputStream(newFile)) { + prop.store(stream, strTitle); + stream.flush(); + } + + if (oldFile.exists()) { + Files.delete(oldFile.toPath()); + } + + if (file.exists()) { + file.renameTo(oldFile); + } + + newFile.renameTo(file); + } catch (final IOException e) { log("PluginBase.saveProp(): " + e.getMessage()); } } - - public void setProp(String strKey, String strValue) throws Exception { - try { - - prop.setProperty(strKey, strValue); - - } catch (Exception e) { - throw new Exception("PluginBase.setProp(): " + e.getMessage()); - } + + /** + * {@link Properties#setProperty(String, String)} + * @param key + * @param value + */ + public void setProp(PropertiesKey key, String value) { + if (key == null || value == null) + return; + + prop.setProperty(key.toString(), value); } - - public String getProp(String strKey) throws Exception { - try { - - return prop.getProperty(strKey); - - } catch (Exception e) { - throw new Exception("PluginBase.getProp(): " + e.getMessage()); - } + + /** + * {@link Properties#getProperty(String)} + */ + public String getProp(PropertiesKey key) { + return key != null ? prop.getProperty(key.toString()) : null; } - - public void setIntProp(String strKey, int nValue) throws Exception { - try { - - prop.setProperty(strKey, String.valueOf(nValue)); - - } catch (Exception e) { - throw new Exception("PluginBase.setIntProp(): " + e.getMessage()); - } + + /** + * {@link Properties#getProperty(String)} + * @param key + * @param value + */ + public void setIntProp(PropertiesKey key, int value) { + final String strValue = String.valueOf(value); + if (key == null || strValue == null) + return; + + setProp(key, strValue); + } + + /** + * get a value and parse it to an integer + * @param key + * @return a parsed number from the key or 0 + */ + public int getIntProp(PropertiesKey key) { + final String value = getProp(key); + return parseInt(value); } - - public int getIntProp(String strKey) throws Exception { - try { - - String strValue = getProp(strKey); - if (strValue != null && !strValue.equals("")) { - return Integer.parseInt(strValue); - } else { - return 0; - } - - } catch (Exception e) { - throw new Exception("PluginBase.getIntProp(): " + e.getMessage()); + + /** + * {@link Integer#parseInt(String)} + * @param value a number as string + * @return a parsed number from the input or 0 + */ + public int parseInt(String value) { + int result = -1; + + if (value != null && !value.trim().isEmpty()) { + try { + result = Integer.parseInt(value); + } catch (final NumberFormatException e) {/* ignore */} } + + return result; } - - protected void removeProp(String strKey) { - try { - - prop.remove(strKey); - - } catch (Exception e) { - log("PluginBase.removeProp(): " + e.getMessage()); - } + + /** + * {@link Properties#remove(Object)} + * @param key + */ + protected void removeProp(PropertiesKey key) { + if (key == null) + return; + + prop.remove(key.toString()); } - + + /** + * {@link Properties#remove(Object)} + * @param key + */ + protected void removeProp(String key) { + if (key == null) + return; + + prop.remove(key); + } + + /** + * {@link Properties#clear()} + */ protected void clearProp() { prop.clear(); } - + + /** + * Returns the Properties + */ + protected Properties getProperties() { + return prop; + } + + /** + * set the timezone to utc + */ protected void setTimezoneUTC() { - dateFormat = new SimpleDateFormat("yyyy.MM.dd_HH.mm_ss"); dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); } + } diff --git a/src/pluginbase/de/todesbaum/util/freenet/fcp2/ClientHello.java b/src/pluginbase/de/todesbaum/util/freenet/fcp2/ClientHello.java index 2f97c4d..fea9ace 100644 --- a/src/pluginbase/de/todesbaum/util/freenet/fcp2/ClientHello.java +++ b/src/pluginbase/de/todesbaum/util/freenet/fcp2/ClientHello.java @@ -33,24 +33,24 @@ * @version $Id$ */ public class ClientHello extends Command { - + /** * The name of the client. */ protected String name; - + /** * The version of the FCP protocol the client expects. */ private String expectedVersion = "2.0"; - + /** * Creates a new ClientHello command. */ protected ClientHello() { super("ClientHello", "ClientHello-" + System.currentTimeMillis()); } - + /** * Returns the value of the ExpectedVersion parameter of this * command. At the moment this value is not used by the node but in the @@ -61,7 +61,7 @@ protected ClientHello() { public String getExpectedVersion() { return expectedVersion; } - + /** * Sets the value of the ExpectedVersion parameter of this * command. At the moment this value is not used by the node but in the @@ -72,7 +72,7 @@ public String getExpectedVersion() { protected void setExpectedVersion(String expectedVersion) { this.expectedVersion = expectedVersion; } - + /** * Returns the name of the client that is connecting. * @@ -81,7 +81,7 @@ protected void setExpectedVersion(String expectedVersion) { public String getName() { return name; } - + /** * Sets the name of the client that is connecting. * @@ -90,7 +90,7 @@ public String getName() { public void setName(String name) { this.name = name; } - + /** * {@inheritDoc} * @@ -101,5 +101,5 @@ protected void write(Writer writer) throws IOException { writer.write("Name=" + name + LINEFEED); writer.write("ExpectedVersion=" + expectedVersion + LINEFEED); } - + } diff --git a/src/pluginbase/de/todesbaum/util/freenet/fcp2/Closer.java b/src/pluginbase/de/todesbaum/util/freenet/fcp2/Closer.java index 942f494..58704e1 100644 --- a/src/pluginbase/de/todesbaum/util/freenet/fcp2/Closer.java +++ b/src/pluginbase/de/todesbaum/util/freenet/fcp2/Closer.java @@ -40,7 +40,7 @@ * @version $Id$ */ public class Closer { - + /** * Closes the given result set. * @@ -51,11 +51,10 @@ public static void close(ResultSet resultSet) { if (resultSet != null) { try { resultSet.close(); - } catch (SQLException ioe1) { - } + } catch (final SQLException ioe1) {} } } - + /** * Closes the given statement. * @@ -66,11 +65,10 @@ public static void close(Statement statement) { if (statement != null) { try { statement.close(); - } catch (SQLException ioe1) { - } + } catch (final SQLException ioe1) {} } } - + /** * Closes the given connection. * @@ -81,11 +79,10 @@ public static void close(Connection connection) { if (connection != null) { try { connection.close(); - } catch (SQLException ioe1) { - } + } catch (final SQLException ioe1) {} } } - + /** * Closes the given server socket. * @@ -96,11 +93,10 @@ public static void close(ServerSocket serverSocket) { if (serverSocket != null) { try { serverSocket.close(); - } catch (IOException ioe1) { - } + } catch (final IOException ioe1) {} } } - + /** * Closes the given socket. * @@ -111,11 +107,10 @@ public static void close(Socket socket) { if (socket != null) { try { socket.close(); - } catch (IOException ioe1) { - } + } catch (final IOException ioe1) {} } } - + /** * Closes the given input stream. * @@ -126,11 +121,10 @@ static void close(InputStream inputStream) { if (inputStream != null) { try { inputStream.close(); - } catch (IOException ioe1) { - } + } catch (final IOException ioe1) {} } } - + /** * Closes the given output stream. * @@ -141,11 +135,10 @@ public static void close(OutputStream outputStream) { if (outputStream != null) { try { outputStream.close(); - } catch (IOException ioe1) { - } + } catch (final IOException ioe1) {} } } - + /** * Closes the given reader. * @@ -156,11 +149,10 @@ public static void close(Reader reader) { if (reader != null) { try { reader.close(); - } catch (IOException ioe1) { - } + } catch (final IOException ioe1) {} } } - + /** * Closes the given writer. * @@ -171,9 +163,8 @@ public static void close(Writer writer) { if (writer != null) { try { writer.close(); - } catch (IOException ioe1) { - } + } catch (final IOException ioe1) {} } } - + } diff --git a/src/pluginbase/de/todesbaum/util/freenet/fcp2/Command.java b/src/pluginbase/de/todesbaum/util/freenet/fcp2/Command.java index 9e910fc..29dcbe7 100644 --- a/src/pluginbase/de/todesbaum/util/freenet/fcp2/Command.java +++ b/src/pluginbase/de/todesbaum/util/freenet/fcp2/Command.java @@ -37,24 +37,24 @@ * @version $Id$ */ public abstract class Command { - + /** * The line feed sequence used by the library. */ static final String LINEFEED = "\r\n"; - + /** * The name of the command. The name is sent to the node so it can not be * chosen arbitrarily! */ private final String commandName; - + /** * The identifier of the command. This identifier is used to identify * replies that are caused by a command. */ private final String identifier; - + /** * Creates a new command with the specified name and identifier. * @@ -65,7 +65,7 @@ public Command(String name, String identifier) { this.commandName = name; this.identifier = identifier; } - + /** * Returns the name of this command. * @@ -74,7 +74,7 @@ public Command(String name, String identifier) { public String getCommandName() { return commandName; } - + /** * Return the identifier of this command. * @@ -83,7 +83,7 @@ public String getCommandName() { public String getIdentifier() { return identifier; } - + /** * Writes all parameters to the specified writer. *

@@ -99,7 +99,7 @@ protected void write(Writer writer) throws IOException { writer.write("Identifier=" + identifier + LINEFEED); } } - + /** * Returns whether this command has payload to send after the message. * Subclasses need to return true here if they need to send @@ -111,7 +111,7 @@ protected void write(Writer writer) throws IOException { protected boolean hasPayload() { return false; } - + /** * Returns the payload of this command as an {@link InputStream}. This * method is never called if {@link #hasPayload()} returns @@ -122,7 +122,7 @@ protected boolean hasPayload() { protected InputStream getPayload() { return null; } - + /** * Returns the length of the payload. This method is never called if * {@link #hasPayload()} returns false. @@ -132,5 +132,5 @@ protected InputStream getPayload() { protected long getPayloadLength() { return -1; } - + } diff --git a/src/pluginbase/de/todesbaum/util/freenet/fcp2/Connection.java b/src/pluginbase/de/todesbaum/util/freenet/fcp2/Connection.java index 0679a98..d921db0 100644 --- a/src/pluginbase/de/todesbaum/util/freenet/fcp2/Connection.java +++ b/src/pluginbase/de/todesbaum/util/freenet/fcp2/Connection.java @@ -30,6 +30,8 @@ import java.util.ArrayList; import java.util.List; +import keepalive.Plugin; + /** * A physical connection to a Freenet node. * @@ -37,57 +39,57 @@ * @version $Id$ */ public class Connection { - + /** * The listeners that receive events from this connection. */ private final List connectionListeners = new ArrayList<>(); - + /** * The node this connection is connected to. */ private final Node node; - + /** * The name of this connection. */ private final String name; - + /** * The network socket of this connection. */ private Socket nodeSocket; - + /** * The input stream that reads from the socket. */ private InputStream nodeInputStream; - + /** * The output stream that writes to the socket. */ private OutputStream nodeOutputStream; - + /** * The thread that reads from the socket. */ private NodeReader nodeReader; - + /** * A writer for the output stream. */ private Writer nodeWriter; - + /** * The NodeHello message sent by the node on connect. */ protected Message nodeHello; - + /** * The temp directory to use. */ private String tempDirectory; - + /** * Creates a new connection to the specified node with the specified name. * @@ -98,7 +100,7 @@ public Connection(Node node, String name) { this.node = node; this.name = name; } - + /** * Adds a listener that gets notified on connection events. * @@ -107,7 +109,7 @@ public Connection(Node node, String name) { public void addConnectionListener(ConnectionListener connectionListener) { connectionListeners.add(connectionListener); } - + /** * Removes a listener from the list of registered listeners. Only the first * matching listener is removed. @@ -118,27 +120,27 @@ public void addConnectionListener(ConnectionListener connectionListener) { public void removeConnectionListener(ConnectionListener connectionListener) { connectionListeners.remove(connectionListener); } - + /** * Notifies listeners about a received message. * * @param message The received message */ protected void fireMessageReceived(Message message) { - for (ConnectionListener connectionListener : connectionListeners) { + for (final ConnectionListener connectionListener : connectionListeners) { connectionListener.messageReceived(this, message); } } - + /** * Notifies listeners about the loss of the connection. */ protected void fireConnectionTerminated() { - for (ConnectionListener connectionListener : connectionListeners) { + for (final ConnectionListener connectionListener : connectionListeners) { connectionListener.connectionTerminated(this); } } - + /** * Returns the name of the connection. * @@ -147,7 +149,7 @@ protected void fireConnectionTerminated() { public String getName() { return name; } - + /** * Sets the temp directory to use for creation of temporary files. * @@ -157,7 +159,7 @@ public String getName() { public void setTempDirectory(String tempDirectory) { this.tempDirectory = tempDirectory; } - + /** * Connects to the node. * @@ -179,27 +181,26 @@ public synchronized boolean connect() throws IOException { nodeOutputStream = nodeSocket.getOutputStream(); nodeWriter = new OutputStreamWriter(nodeOutputStream, Charset.forName("UTF-8")); nodeReader = new NodeReader(nodeInputStream); - Thread nodeReaderThread = new Thread(nodeReader); + final Thread nodeReaderThread = new Thread(nodeReader); nodeReaderThread.setDaemon(true); - nodeReaderThread.setName("KeepAlive FCP Thread"); + nodeReaderThread.setName(Plugin.PLUGIN_NAME + " FCP Thread"); nodeReaderThread.start(); - ClientHello clientHello = new ClientHello(); + final ClientHello clientHello = new ClientHello(); clientHello.setName(name); clientHello.setExpectedVersion("2.0"); execute(clientHello); synchronized (this) { try { wait(10000); - } catch (InterruptedException e) { - } + } catch (final InterruptedException e) {} } return nodeHello != null; - } catch (IOException ioe1) { + } catch (final IOException ioe1) { disconnect(); throw ioe1; } } - + /** * Returns whether this connection is still connected to the node. * @@ -209,7 +210,7 @@ public synchronized boolean connect() throws IOException { public boolean isConnected() { return (nodeHello != null) && (nodeSocket != null) && (nodeSocket.isConnected()); } - + /** * Returns the NodeHello message the node sent on connection. * @@ -218,7 +219,7 @@ public boolean isConnected() { public Message getNodeHello() { return nodeHello; } - + /** * Disconnects from the node. */ @@ -226,29 +227,25 @@ public void disconnect() { if (nodeWriter != null) { try { nodeWriter.close(); - } catch (IOException ioe1) { - } + } catch (final IOException ioe1) {} nodeWriter = null; } if (nodeOutputStream != null) { try { nodeOutputStream.close(); - } catch (IOException ioe1) { - } + } catch (final IOException ioe1) {} nodeOutputStream = null; } if (nodeInputStream != null) { try { nodeInputStream.close(); - } catch (IOException ioe1) { - } + } catch (final IOException ioe1) {} nodeInputStream = null; } if (nodeSocket != null) { try { nodeSocket.close(); - } catch (IOException ioe1) { - } + } catch (final IOException ioe1) {} nodeSocket = null; } synchronized (this) { @@ -256,7 +253,7 @@ public void disconnect() { } fireConnectionTerminated(); } - + /** * Executes the specified command. * @@ -283,7 +280,7 @@ public synchronized void execute(Command command) throws IllegalStateException, nodeOutputStream.flush(); } } - + /** * The reader thread for this connection. This is essentially a thread that * reads lines from the node, creates messages from them and notifies @@ -293,13 +290,13 @@ public synchronized void execute(Command command) throws IllegalStateException, * @version $Id$ */ protected class NodeReader implements Runnable { // changed by jeriadoc 11/2011: private -> protected - + /** * The input stream to read from. */ @SuppressWarnings("hiding") private final InputStream nodeInputStream; - + /** * Creates a new reader that reads from the specified input stream. * @@ -308,7 +305,7 @@ protected class NodeReader implements Runnable { // changed by jeriadoc 1 public NodeReader(InputStream nodeInputStream) { this.nodeInputStream = nodeInputStream; } - + /** * Main loop of the reader. Lines are read and converted into * {@link Message} objects. @@ -337,16 +334,16 @@ public void run() { tempFile = File.createTempFile("fcpv2", "data", (tempDirectory != null) ? new File(tempDirectory) : null); tempFile.deleteOnExit(); try (FileOutputStream tempFileOutputStream = new FileOutputStream(tempFile)) { - long dataLength = Long.parseLong(message.get("DataLength")); + final long dataLength = Long.parseLong(message.get("DataLength")); StreamCopier.copy(nodeInputStream, tempFileOutputStream, dataLength); } message.setPayloadInputStream(new TempFileInputStream(tempFile)); - } catch (IOException ioe1) { + } catch (final IOException ioe1) { ioe1.printStackTrace(); } } if ("Data".equals(line) || "EndMessage".equals(line)) { - if (message.getName().equals("NodeHello")) { + if ("NodeHello".equals(message.getName())) { nodeHello = message; synchronized (Connection.this) { Connection.this.notify(); @@ -357,11 +354,11 @@ public void run() { message = null; continue; } - int equalsPosition = line.indexOf('='); + final int equalsPosition = line.indexOf('='); if (equalsPosition > -1) { - String key = line.substring(0, equalsPosition).trim(); - String value = line.substring(equalsPosition + 1).trim(); - if (key.equals("Identifier")) { + final String key = line.substring(0, equalsPosition).trim(); + final String value = line.substring(equalsPosition + 1).trim(); + if ("Identifier".equals(key)) { message.setIdentifier(value); } else { message.put(key, value); @@ -375,25 +372,23 @@ public void run() { /* if we got here, some error occured! */ throw new IOException("Unexpected line: " + line); } - } catch (IOException ioe1) { + } catch (final IOException ioe1) { // ioe1.printStackTrace(); } finally { if (nodeReader != null) { try { nodeReader.close(); - } catch (IOException ioe1) { - } + } catch (final IOException ioe1) {} } if (nodeInputStream != null) { try { nodeInputStream.close(); - } catch (IOException ioe1) { - } + } catch (final IOException ioe1) {} } } Connection.this.disconnect(); } - + } - + } diff --git a/src/pluginbase/de/todesbaum/util/freenet/fcp2/ConnectionListener.java b/src/pluginbase/de/todesbaum/util/freenet/fcp2/ConnectionListener.java index 0dcc9df..5a9e4ac 100644 --- a/src/pluginbase/de/todesbaum/util/freenet/fcp2/ConnectionListener.java +++ b/src/pluginbase/de/todesbaum/util/freenet/fcp2/ConnectionListener.java @@ -27,7 +27,7 @@ * @version $Id$ */ public interface ConnectionListener extends EventListener { - + /** * Notifies a client that a message was received. * @@ -35,12 +35,12 @@ public interface ConnectionListener extends EventListener { * @param message The message that was received */ void messageReceived(Connection connection, Message message); - + /** * Notifies a client that the connection to the node has been lost. * * @param connection The connection that was lost */ void connectionTerminated(Connection connection); - + } diff --git a/src/pluginbase/de/todesbaum/util/freenet/fcp2/LineInputStream.java b/src/pluginbase/de/todesbaum/util/freenet/fcp2/LineInputStream.java index 09c3e80..3694e55 100644 --- a/src/pluginbase/de/todesbaum/util/freenet/fcp2/LineInputStream.java +++ b/src/pluginbase/de/todesbaum/util/freenet/fcp2/LineInputStream.java @@ -29,17 +29,17 @@ * @version $Id$ */ public class LineInputStream extends FilterInputStream { - + private boolean skipLinefeed; private final StringBuffer lineBuffer = new StringBuffer(); - + /** * @param in */ public LineInputStream(InputStream in) { super(in); } - + public synchronized String readLine() { try { lineBuffer.setLength(0); @@ -61,9 +61,9 @@ public synchronized String readLine() { } } return lineBuffer.toString(); - } catch (IOException e) { + } catch (final IOException e) { return null; } } - + } diff --git a/src/pluginbase/de/todesbaum/util/freenet/fcp2/Message.java b/src/pluginbase/de/todesbaum/util/freenet/fcp2/Message.java index 495962f..898a7aa 100644 --- a/src/pluginbase/de/todesbaum/util/freenet/fcp2/Message.java +++ b/src/pluginbase/de/todesbaum/util/freenet/fcp2/Message.java @@ -21,8 +21,8 @@ import java.io.InputStream; import java.util.HashMap; import java.util.Map; -import java.util.Set; import java.util.Map.Entry; +import java.util.Set; /** * Contains replies sent by the Freenet node. A message always has a name, and @@ -35,27 +35,27 @@ * @see de.todesbaum.util.freenet.fcp2.Client */ public class Message { - + /** * The name of this message. */ private final String name; - + /** * The identifier of this message. */ private String identifier = ""; - + /** * The parameters of this message. */ private final Map parameters = new HashMap<>(); - + /** * The payload. */ private InputStream payloadInputStream; - + /** * Creates a new message with the specified name. * @@ -64,7 +64,7 @@ public class Message { public Message(String name) { this.name = name; } - + /** * Returns the identifier of this message. * @@ -73,7 +73,7 @@ public Message(String name) { public String getIdentifier() { return identifier; } - + /** * Sets the identifier of this message. * @@ -82,7 +82,7 @@ public String getIdentifier() { public void setIdentifier(String identifier) { this.identifier = identifier; } - + /** * Returns the name of this message. * @@ -91,7 +91,7 @@ public void setIdentifier(String identifier) { public String getName() { return name; } - + /** * Tests whether this message contains the parameter with the specified key. * Key names are compared ignoring case. @@ -103,7 +103,7 @@ public String getName() { public boolean containsKey(String key) { return parameters.containsKey(key.toLowerCase()); } - + /** * Returns all parameters of this message. The keys of the entries are all * lower case so if you want to match the parameter names you have to watch @@ -114,7 +114,7 @@ public boolean containsKey(String key) { public Set> entrySet() { return parameters.entrySet(); } - + /** * Returns the value of the parameter with the name specified by * key. @@ -125,7 +125,7 @@ public Set> entrySet() { public String get(String key) { return parameters.get(key.toLowerCase()); } - + /** * Stores the specified value as parameter with the name specified by * key. @@ -138,7 +138,7 @@ public String get(String key) { public String put(String key, String value) { return parameters.put(key.toLowerCase(), value); } - + /** * Returns the number of parameters in this message. * @@ -147,21 +147,21 @@ public String put(String key, String value) { public int size() { return parameters.size(); } - + /** * @return Returns the payloadInputStream. */ public InputStream getPayloadInputStream() { return payloadInputStream; } - + /** * @param payloadInputStream The payloadInputStream to set. */ public void setPayloadInputStream(InputStream payloadInputStream) { this.payloadInputStream = payloadInputStream; } - + /** * Returns a textual representation of this message, containing its name, * the identifier, and the parameters. @@ -172,5 +172,5 @@ public void setPayloadInputStream(InputStream payloadInputStream) { public String toString() { return name + "[identifier=" + identifier + ",parameters=" + parameters.toString() + "]"; } - + } diff --git a/src/pluginbase/de/todesbaum/util/freenet/fcp2/Node.java b/src/pluginbase/de/todesbaum/util/freenet/fcp2/Node.java index e27668b..67df2d3 100644 --- a/src/pluginbase/de/todesbaum/util/freenet/fcp2/Node.java +++ b/src/pluginbase/de/todesbaum/util/freenet/fcp2/Node.java @@ -25,22 +25,22 @@ * @version $Id$ */ public class Node { - + /** * The default port of FCPv2. */ public static final int DEFAULT_PORT = 9481; - + /** * The hostname of the node. */ protected String hostname; - + /** * The port number of the node. */ protected int port; - + /** * Creates a new node with the specified hostname and the default port * number. @@ -51,7 +51,7 @@ public class Node { public Node(String hostname) { this(hostname, DEFAULT_PORT); } - + /** * Creates a new node with the specified hostname and port number. * @@ -62,7 +62,7 @@ public Node(String hostname, int port) { this.hostname = hostname; this.port = port; } - + /** * Returns the hostname of the node. * @@ -71,7 +71,7 @@ public Node(String hostname, int port) { public String getHostname() { return hostname; } - + /** * Returns the port number of the node. * @@ -80,5 +80,5 @@ public String getHostname() { public int getPort() { return port; } - + } diff --git a/src/pluginbase/de/todesbaum/util/freenet/fcp2/StreamCopier.java b/src/pluginbase/de/todesbaum/util/freenet/fcp2/StreamCopier.java index c3bcb9b..7b5d0d2 100644 --- a/src/pluginbase/de/todesbaum/util/freenet/fcp2/StreamCopier.java +++ b/src/pluginbase/de/todesbaum/util/freenet/fcp2/StreamCopier.java @@ -27,32 +27,32 @@ * @version $Id$ */ public class StreamCopier { - + /** * The default size of the buffer. */ private static final int BUFFER_SIZE = 64 * 1024; - + /** * The {@link InputStream} to read from. */ - private InputStream inputStream; - + private final InputStream inputStream; + /** * The {@link OutputStream} to write to. */ - private OutputStream outputStream; - + private final OutputStream outputStream; + /** * The number of bytes to copy. */ - private long length; - + private final long length; + /** * The size of the buffer. */ - private int bufferSize; - + private final int bufferSize; + /** * Creates a new StreamCopier with the specified parameters and the default * buffer size. @@ -64,7 +64,7 @@ public class StreamCopier { public StreamCopier(InputStream inputStream, OutputStream outputStream, long length) { this(inputStream, outputStream, length, BUFFER_SIZE); } - + /** * Creates a new StreamCopier with the specified parameters and the default * buffer size. @@ -80,7 +80,7 @@ public StreamCopier(InputStream inputStream, OutputStream outputStream, long len this.length = length; this.bufferSize = bufferSize; } - + /** * Copies the stream data. If the input stream is depleted before the * requested number of bytes have been read an {@link EOFException} is @@ -93,7 +93,7 @@ public StreamCopier(InputStream inputStream, OutputStream outputStream, long len public void copy() throws EOFException, IOException { copy(inputStream, outputStream, length, bufferSize); } - + /** * Copies length bytes from the inputStream to the * outputStream. @@ -106,7 +106,7 @@ public void copy() throws EOFException, IOException { public static void copy(InputStream inputStream, OutputStream outputStream, long length) throws IOException { copy(inputStream, outputStream, length, BUFFER_SIZE); } - + /** * Copies length bytes from the inputStream to the * outputStream using a buffer with the specified size @@ -119,9 +119,9 @@ public static void copy(InputStream inputStream, OutputStream outputStream, long */ public static void copy(InputStream inputStream, OutputStream outputStream, long length, int bufferSize) throws IOException { long remaining = length; - byte[] buffer = new byte[bufferSize]; + final byte[] buffer = new byte[bufferSize]; while (remaining > 0) { - int read = inputStream.read(buffer, 0, (int) Math.min(Integer.MAX_VALUE, Math.min(bufferSize, remaining))); + final int read = inputStream.read(buffer, 0, (int) Math.min(Integer.MAX_VALUE, Math.min(bufferSize, remaining))); if (read == -1) { throw new EOFException(); } @@ -129,5 +129,5 @@ public static void copy(InputStream inputStream, OutputStream outputStream, long remaining -= read; } } - + } diff --git a/src/pluginbase/de/todesbaum/util/freenet/fcp2/TempFileInputStream.java b/src/pluginbase/de/todesbaum/util/freenet/fcp2/TempFileInputStream.java index f691260..642a93a 100644 --- a/src/pluginbase/de/todesbaum/util/freenet/fcp2/TempFileInputStream.java +++ b/src/pluginbase/de/todesbaum/util/freenet/fcp2/TempFileInputStream.java @@ -28,9 +28,9 @@ * @version $Id$ */ public class TempFileInputStream extends FileInputStream { - - private File tempFile; - + + private final File tempFile; + /** * @param name * @throws FileNotFoundException @@ -38,7 +38,7 @@ public class TempFileInputStream extends FileInputStream { public TempFileInputStream(String name) throws FileNotFoundException { this(new File(name)); } - + /** * @param file * @throws FileNotFoundException @@ -47,11 +47,11 @@ protected TempFileInputStream(File file) throws FileNotFoundException { super(file); tempFile = file; } - + @Override public void close() throws IOException { super.close(); tempFile.delete(); } - + } diff --git a/src/resources/static/style.css b/src/resources/static/style.css index 330ab31..166264c 100644 --- a/src/resources/static/style.css +++ b/src/resources/static/style.css @@ -157,3 +157,11 @@ input[type="text"] { padding-bottom: 8em; } } + +/* for the entry table anchors, because the freenet header */ +a.anchor { + display: block; + position: relative; + top: -100px; + visibility: hidden; +} diff --git a/src/resources/templates/overview_table.html b/src/resources/templates/overview_table.html new file mode 100644 index 0000000..9bcbd62 --- /dev/null +++ b/src/resources/templates/overview_table.html @@ -0,0 +1,12 @@ + + + + + + + + + + + ${url_entries} +
URItotal
blocks
available
blocks
missed
blocks
blocks
availability
segments
availability
Actions
diff --git a/src/resources/templates/properties.html b/src/resources/templates/properties.html index 6a08bb0..0cb4775 100644 --- a/src/resources/templates/properties.html +++ b/src/resources/templates/properties.html @@ -40,7 +40,18 @@ - + + + + + + + + Remove: + + + + diff --git a/src/resources/templates/url_entry.html b/src/resources/templates/url_entry.html new file mode 100644 index 0000000..cea1556 --- /dev/null +++ b/src/resources/templates/url_entry.html @@ -0,0 +1,15 @@ + + + + ${url_short} + + ${url_blockSize} + ${url_success} + ${url_failure} + ${url_persistence} % + ${url_segmentsAvailability} % + remove + log + ${url_modus} + ${url_active} +