Merge pull request #43 from RatzzFatzz/dev

Add date filter & incremental updating

pom.xml

@@ -4,7 +4,7 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
-    <groupId>MKVAudioSubtileChanger</groupId>
+    <groupId>at.pcgamingfreaks</groupId>
     <artifactId>MKVAudioSubtitleChanger</artifactId>
     <version>3.0</version>
 
@@ -20,7 +20,7 @@
                 <directory>./</directory>
                 <includes>
                     <include>language-codes</include>
-                    <include>version.properties</include>
+                    <include>project.properties</include>
                 </includes>
                 <filtering>true</filtering>
             </resource>
@@ -194,6 +194,12 @@
             <artifactId>YAML-Parser</artifactId>
             <version>2.0-SNAPSHOT</version>
         </dependency>
+        <!-- https://mvnrepository.com/artifact/net.harawata/appdirs -->
+        <dependency>
+            <groupId>net.harawata</groupId>
+            <artifactId>appdirs</artifactId>
+            <version>1.2.1</version>
+        </dependency>
     </dependencies>
 
 </project>

project.properties

@@ -0,0 +1,2 @@
+version=${project.version}
+project_name=${project.artifactId}

src/main/java/at/pcgamingfreaks/mkvaudiosubtitlechanger/config/Config.java

@@ -38,6 +38,8 @@ public class Config {
 
     private Integer coherent;
     private boolean forceCoherent;
+    private boolean onlyNewFiles;
+    private Date filterDate;
 
     private Set<String> forcedKeywords = new HashSet<>(Arrays.asList("forced", "signs", "songs"));
     private Set<String> commentaryKeywords = new HashSet<>(Arrays.asList("commentary", "director"));
@@ -67,20 +69,29 @@ public class Config {
                     mkvToolNix.getAbsolutePath() + "/" + application;
     }
 
+    public String getNormalizedLibraryPath() {
+        return this.getLibraryPath().getAbsolutePath().replace("\\", "/");
+    }
+
     @Override
     public String toString() {
         return new StringJoiner(", ", Config.class.getSimpleName() + "[", "]")
-                .add("parser=" + parser).add("\n")
+                .add("parser=" + parser)
-                .add("formatter=" + formatter).add("\n")
+                .add("formatter=" + formatter)
-                .add("configPath=" + configPath).add("\n")
+                .add("configPath=" + configPath)
-                .add("libraryPath=" + libraryPath).add("\n")
+                .add("libraryPath=" + libraryPath)
-                .add("isSafeMode=" + safeMode).add("\n")
+                .add("mkvToolNix=" + mkvToolNix)
-                .add("forcedKeywords=" + forcedKeywords).add("\n")
+                .add("threads=" + threads)
-                .add("commentaryKeywords=" + commentaryKeywords).add("\n")
+                .add("includePattern=" + includePattern)
-                .add("excludedDirectories=" + excludedDirectories).add("\n")
+                .add("safeMode=" + safeMode)
-                .add("threadCount=" + threads).add("\n")
+                .add("coherent=" + coherent)
-                .add("includePattern=" + includePattern).add("\n")
+                .add("forceCoherent=" + forceCoherent)
-                .add("mkvToolNixPath='" + mkvToolNix + "'").add("\n")
+                .add("onlyNewFiles=" + onlyNewFiles)
+                .add("filterDate=" + filterDate)
+                .add("forcedKeywords=" + forcedKeywords)
+                .add("commentaryKeywords=" + commentaryKeywords)
+                .add("excludedDirectories=" + excludedDirectories)
+                .add("preferredSubtitles=" + preferredSubtitles)
                 .add("attributeConfig=" + attributeConfig)
                 .toString();
     }

src/main/java/at/pcgamingfreaks/mkvaudiosubtitlechanger/config/ConfigLoader.java

@@ -2,7 +2,7 @@ package at.pcgamingfreaks.mkvaudiosubtitlechanger.config;
 
 import at.pcgamingfreaks.mkvaudiosubtitlechanger.config.validator.*;
 import at.pcgamingfreaks.mkvaudiosubtitlechanger.model.ConfigProperty;
-import at.pcgamingfreaks.mkvaudiosubtitlechanger.util.VersionUtil;
+import at.pcgamingfreaks.mkvaudiosubtitlechanger.util.ProjectUtil;
 import at.pcgamingfreaks.yaml.YAML;
 import at.pcgamingfreaks.yaml.YamlInvalidContentException;
 import org.apache.commons.cli.*;
@@ -11,19 +11,24 @@ import java.io.IOException;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Comparator;
 import java.util.List;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static at.pcgamingfreaks.mkvaudiosubtitlechanger.model.ConfigProperty.*;
 import static at.pcgamingfreaks.mkvaudiosubtitlechanger.util.CommandLineOptionsUtil.optionOf;
 
 public class ConfigLoader {
-    private static final List<ConfigValidator<?>> VALIDATORS = List.of(
+    private static final List<ConfigValidator<?>> VALIDATORS = Stream.of(
             new ConfigPathValidator(CONFIG_PATH, false),
             new PathValidator(LIBRARY, true, null),
             new ThreadValidator(THREADS, false, 2),
             new MkvToolNixPathValidator(MKV_TOOL_NIX, true, Path.of("C:\\Program Files\\MKVToolNix").toFile()),
             new BooleanValidator(SAFE_MODE, false),
+            new BooleanValidator(ONLY_NEW_FILES, false),
+            new DateValidator(FILTER_DATE, false),
             new PatternValidator(INCLUDE_PATTERN, false, Pattern.compile(".*")),
             new SetValidator(FORCED_KEYWORDS, false, true),
             new SetValidator(COMMENTARY_KEYWORDS, false, true),
@@ -32,7 +37,7 @@ public class ConfigLoader {
             new AttributeConfigValidator(),
             new CoherentConfigValidator(COHERENT, false),
             new BooleanValidator(FORCE_COHERENT, false)
-    );
+    ).sorted(Comparator.comparing((ConfigValidator<?> validator) -> validator.getWeight()).reversed()).collect(Collectors.toList());
 
     public static void initConfig(String[] args) {
         HelpFormatter formatter = new HelpFormatter();
@@ -98,7 +103,7 @@ public class ConfigLoader {
 
     private static void exitIfVersion(CommandLine cmd) {
         if (cmd.hasOption(VERSION.prop())) {
-            System.out.printf("MKV Audio Subtitle Changer Version %s%n", VersionUtil.getVersion());
+            System.out.printf("MKV Audio Subtitle Changer Version %s%n", ProjectUtil.getVersion());
             System.exit(0);
         }
     }

src/main/java/at/pcgamingfreaks/mkvaudiosubtitlechanger/config/validator/ConfigPathValidator.java

@@ -27,4 +27,9 @@ public class ConfigPathValidator extends PathValidator {
     protected boolean isValid(File result) {
         return super.isValid(result) && (result.getAbsolutePath().endsWith(".yml") || result.getAbsolutePath().endsWith(".yaml"));
     }
+
+    @Override
+    public int getWeight() {
+        return 100;
+    }
 }

src/main/java/at/pcgamingfreaks/mkvaudiosubtitlechanger/config/validator/ConfigValidator.java

@@ -6,18 +6,18 @@ import at.pcgamingfreaks.mkvaudiosubtitlechanger.model.ConfigProperty;
 import at.pcgamingfreaks.yaml.YAML;
 import at.pcgamingfreaks.yaml.YamlKeyNotFoundException;
 import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.lang3.StringUtils;
 
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
-import java.util.Arrays;
-import java.util.List;
 import java.util.Optional;
+import java.util.StringJoiner;
 import java.util.function.BiFunction;
 import java.util.function.Predicate;
-import java.util.stream.Collectors;
 
+@Slf4j
 @RequiredArgsConstructor
 public abstract class ConfigValidator<FieldType> {
     protected final ConfigProperty property;
@@ -38,7 +38,9 @@ public abstract class ConfigValidator<FieldType> {
         Optional<FieldType> cmdResult = provideDataCmd().apply(cmd, property);
         Optional<FieldType> yamlResult = provideDataYaml().apply(yaml, property);
 
-		if (cmdResult.isPresent()) {
+        if (isOverwritingNecessary()) {
+            result = overwriteValue();
+        } else if (cmdResult.isPresent()) {
             result = cmdResult.get();
         } else if (yamlResult.isPresent()) {
             result = yamlResult.get();
@@ -98,6 +100,20 @@ public abstract class ConfigValidator<FieldType> {
         };
     }
 
+    /**
+     * @return true if overwriting this property is necessary.
+     */
+    protected boolean isOverwritingNecessary() {
+        return false;
+    }
+
+    /**
+     * @return {@link FieldType} to overwrite result with.
+     */
+    protected FieldType overwriteValue() {
+        return null;
+    }
+
     /**
      * Parse input parameter to desired format.
      *
@@ -145,5 +161,16 @@ public abstract class ConfigValidator<FieldType> {
                 && StringUtils.equalsIgnoreCase(method.getName().replace("get", ""), property.prop().replace("-", ""));
     }
 
-
+    public int getWeight() {
+        return 50;
+    }
+
+    @Override
+    public String toString() {
+        return new StringJoiner(", ", ConfigValidator.class.getSimpleName() + "[", "]")
+                .add("property=" + property)
+                .add("required=" + required)
+                .add("defaultValue=" + defaultValue)
+                .toString();
+    }
 }

src/main/java/at/pcgamingfreaks/mkvaudiosubtitlechanger/config/validator/DateValidator.java

@@ -0,0 +1,61 @@
+package at.pcgamingfreaks.mkvaudiosubtitlechanger.config.validator;
+
+import at.pcgamingfreaks.mkvaudiosubtitlechanger.config.Config;
+import at.pcgamingfreaks.mkvaudiosubtitlechanger.model.ConfigProperty;
+import at.pcgamingfreaks.mkvaudiosubtitlechanger.util.DateUtils;
+import at.pcgamingfreaks.mkvaudiosubtitlechanger.util.ProjectUtil;
+import at.pcgamingfreaks.yaml.YAML;
+import at.pcgamingfreaks.yaml.YamlInvalidContentException;
+import lombok.extern.slf4j.Slf4j;
+import net.harawata.appdirs.AppDirsFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.Date;
+
+@Slf4j
+public class DateValidator extends ConfigValidator<Date> {
+    private static final Date INVALID_DATE = new Date(0);
+    private static final Date DEFAULT_DATE = new Date(1000);
+
+    public DateValidator(ConfigProperty property, boolean required) {
+        super(property, required, null);
+    }
+
+    @Override
+    protected boolean isOverwritingNecessary() {
+        return Config.getInstance().isOnlyNewFiles();
+    }
+
+    @Override
+    protected Date overwriteValue() {
+        try {
+            String filePath = AppDirsFactory.getInstance().getUserConfigDir(ProjectUtil.getProjectName(), null, null);
+            File lastExecutionFile = Path.of(filePath + "/last-execution.yml").toFile();
+            if (!lastExecutionFile.exists()) {
+                return DEFAULT_DATE;
+            }
+            YAML yaml = new YAML(lastExecutionFile);
+            return parse(yaml.getString(Config.getInstance().getNormalizedLibraryPath(), DateUtils.convert(DEFAULT_DATE)));
+        } catch (YamlInvalidContentException | IOException e) {
+            log.error("Couldn't open last-execution.properties");
+            return INVALID_DATE;
+        }
+    }
+
+    @Override
+    Date parse(String value) {
+        return DateUtils.convert(value, INVALID_DATE);
+    }
+
+    @Override
+    boolean isValid(Date result) {
+        return !result.equals(INVALID_DATE);
+    }
+
+    @Override
+    public int getWeight() {
+        return 40;
+    }
+}

src/main/java/at/pcgamingfreaks/mkvaudiosubtitlechanger/impl/FileFilter.java

@@ -2,18 +2,50 @@ package at.pcgamingfreaks.mkvaudiosubtitlechanger.impl;
 
 import at.pcgamingfreaks.mkvaudiosubtitlechanger.config.Config;
 import at.pcgamingfreaks.mkvaudiosubtitlechanger.model.ResultStatistic;
+import at.pcgamingfreaks.mkvaudiosubtitlechanger.util.DateUtils;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 
 import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Date;
 
+@Slf4j
 public class FileFilter {
     static boolean accept(File pathName, String[] fileExtensions) {
-        if (StringUtils.endsWithAny(pathName.getAbsolutePath().toLowerCase(), fileExtensions)
+        if (hasProperFileExtension(pathName, fileExtensions)
-                && Config.getInstance().getIncludePattern().matcher(pathName.getName()).matches()) {
+                && hasMatchingPattern(pathName)
+                && isNewer(pathName)) {
             return true;
         }
 
         ResultStatistic.getInstance().excluded();
         return false;
     }
+
+    private static boolean hasProperFileExtension(File pathName, String[] fileExtensions) {
+        return StringUtils.endsWithAny(pathName.getAbsolutePath().toLowerCase(), fileExtensions);
+    }
+
+    private static boolean hasMatchingPattern(File pathName) {
+        return Config.getInstance().getIncludePattern().matcher(pathName.getName()).matches();
+    }
+
+    private static boolean isNewer(File pathName) {
+        Config config = Config.getInstance();
+        if (config.getFilterDate() == null) return true;
+        try {
+            BasicFileAttributes attributes = Files.readAttributes(pathName.toPath(), BasicFileAttributes.class);
+            return isNewer(DateUtils.convert(attributes.creationTime().toMillis()));
+        } catch (IOException e) {
+            log.warn("File attributes could not be read.", e);
+        }
+        return true;
+    }
+
+    private static boolean isNewer(Date creationDate) {
+        return creationDate.toInstant().isAfter(Config.getInstance().getFilterDate().toInstant());
+    }
 }

src/main/java/at/pcgamingfreaks/mkvaudiosubtitlechanger/impl/kernel/AttributeUpdaterKernel.java

@@ -8,16 +8,23 @@ import at.pcgamingfreaks.mkvaudiosubtitlechanger.model.AttributeConfig;
 import at.pcgamingfreaks.mkvaudiosubtitlechanger.model.FileAttribute;
 import at.pcgamingfreaks.mkvaudiosubtitlechanger.model.FileInfoDto;
 import at.pcgamingfreaks.mkvaudiosubtitlechanger.model.ResultStatistic;
+import at.pcgamingfreaks.mkvaudiosubtitlechanger.util.DateUtils;
+import at.pcgamingfreaks.mkvaudiosubtitlechanger.util.ProjectUtil;
+import at.pcgamingfreaks.yaml.YAML;
+import at.pcgamingfreaks.yaml.YamlInvalidContentException;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import me.tongfei.progressbar.ProgressBar;
 import me.tongfei.progressbar.ProgressBarBuilder;
 import me.tongfei.progressbar.ProgressBarStyle;
+import net.harawata.appdirs.AppDirsFactory;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.Path;
 import java.util.Collection;
+import java.util.Date;
 import java.util.List;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -57,6 +64,8 @@ public abstract class AttributeUpdaterKernel {
             executor.awaitTermination(1, TimeUnit.DAYS);
         }
 
+        endProcess();
+
         statistic.stopTimer();
         statistic.printResult();
     }
@@ -139,4 +148,26 @@ public abstract class AttributeUpdaterKernel {
             log.warn("File couldn't be updated: '{}', Error: {}", fileInfo.getFile().getAbsoluteFile(), e.getMessage());
         }
     }
+
+    protected void endProcess() {
+        if (Config.getInstance().isSafeMode()) {
+            return;
+        }
+
+        try {
+            String filePath = AppDirsFactory.getInstance().getUserConfigDir(ProjectUtil.getProjectName(), null, null);
+
+            File configDir = Path.of(filePath).toFile();
+            if (!configDir.exists()) configDir.mkdirs();
+
+            File lastExecutionFile = Path.of(filePath + "/last-execution.yml").toFile();
+            if (!lastExecutionFile.exists()) lastExecutionFile.createNewFile();
+
+            YAML yaml = new YAML(lastExecutionFile);
+            yaml.set(Config.getInstance().getNormalizedLibraryPath(), DateUtils.convert(new Date()));
+            yaml.save(lastExecutionFile);
+        } catch (IOException | YamlInvalidContentException e) {
+            log.error("last-execution.yml could not be created or read.", e);
+        }
+    }
 }

src/main/java/at/pcgamingfreaks/mkvaudiosubtitlechanger/model/ConfigProperty.java

@@ -18,6 +18,8 @@ public enum ConfigProperty {
     COHERENT("coherent", "Try to match all files in dir of depth with the same config", "c", 1),
     FORCE_COHERENT("force-coherent", "Force coherent and don't update anything if config fits not whole config (default: false)", "cf", 0),
     WINDOWS("windows", "Is operating system windows", null, 0),
+    ONLY_NEW_FILES("only-new-files", "Sets filter-date to last successful execution (Overwrites input of filter-date)", "n", 0),
+    FILTER_DATE("filter-date", "Only consider files created newer than entered date (format: \"dd.MM.yyyy-HH:mm:ss\")", "d", 1),
     THREADS("threads", "Thread count (default: 2)", "t", 1),
     INCLUDE_PATTERN("include-pattern", "Include files matching pattern (default: \".*\")", "i", 1),
     EXCLUDED_DIRECTORY("excluded-directories", "Directories to be excluded, combines with config file", "e", Option.UNLIMITED_VALUES),

src/main/java/at/pcgamingfreaks/mkvaudiosubtitlechanger/util/DateUtils.java

@@ -0,0 +1,29 @@
+package at.pcgamingfreaks.mkvaudiosubtitlechanger.util;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+public class DateUtils {
+    private static final SimpleDateFormat dateFormat = new SimpleDateFormat("dd.MM.yyyy-HH:mm:ss");
+
+    public static Date convert(long millis) {
+        return new Date(millis);
+    }
+
+    /**
+     * Convert String to date.
+     * @return parsed date, defaultDate if exception occurs
+     */
+    public static Date convert(String date, Date defaultDate) {
+        try {
+            return dateFormat.parse(date);
+        } catch (ParseException e) {
+            return defaultDate;
+        }
+    }
+
+    public static String convert(Date date) {
+        return dateFormat.format(date);
+    }
+}

src/main/java/at/pcgamingfreaks/mkvaudiosubtitlechanger/util/ProjectUtil.java

@@ -0,0 +1,25 @@
+package at.pcgamingfreaks.mkvaudiosubtitlechanger.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+
+public class ProjectUtil {
+    private static final Properties PROJECT_PROPERTIES = new Properties();
+
+    static {
+        try (InputStream propertiesStream = ProjectUtil.class.getClassLoader().getResourceAsStream("project.properties")) {
+            PROJECT_PROPERTIES.load(propertiesStream);
+        } catch (IOException e) {
+            throw new RuntimeException(e.getMessage());
+        }
+    }
+
+    public static String getVersion() {
+        return PROJECT_PROPERTIES.getProperty("version");
+    }
+
+    public static String getProjectName() {
+        return  PROJECT_PROPERTIES.getProperty("project_name");
+    }
+}

src/main/java/at/pcgamingfreaks/mkvaudiosubtitlechanger/util/VersionUtil.java

@@ -1,18 +0,0 @@
-package at.pcgamingfreaks.mkvaudiosubtitlechanger.util;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Properties;
-
-public class VersionUtil {
-    public static String getVersion() {
-        try (InputStream propertiesStream = VersionUtil.class.getClassLoader().getResourceAsStream("version.properties")) {
-            Properties properties = new Properties();
-            properties.load(propertiesStream);
-
-            return properties.getProperty("version");
-        } catch (IOException e) {
-            throw new RuntimeException(e.getMessage());
-        }
-    }
-}

version.properties

@@ -1 +0,0 @@
-version=${project.version}